"""
Interactive Spatial Monitoring (Mapobs)
=======================================

The ``mapobs`` module is designed to visualize the geographic distribution of observations. Beyond generating static high-resolution global maps, it automates the creation of an **Interactive Web Session** to analyze data evolution across 15-minute time bins.

Usage & Command-Line Parameters
-------------------------------

.. tip::
   **Interactive Session Setup:** Generating cross-sections across massive vertical and horizontal bins is computationally heavy. Always start an interactive session:
   
   .. code-block:: bash
   
       qsub -I -X -l select=4:ncpus=80:mpiprocs=80:ompthreads=1:mem=185gb -l place=scatter -l walltime=6:0:0


To generate a global observation session (e.g., evaluating AMSU-A satellite coverage):

.. code-block:: bash

    python -c 'import pikobs; pikobs.mapobs.arg_call()' \\
         --path_experience_files  /home/dlo001/sites8/Data_pikobs/monitoring1/ \\
         --experience_name        Experience \\
         --pathwork               /home/dlo001/sites8/pikobsmap_session \\
         --datestart              2026033000 \\
         --dateend                2026033012 \\
         --region                 Monde \\
         --family                 amsua \\
         --varnos                 1 \\
         --flags_criteria         assimilee \\
         --n_cpus                 40

**What this command does:**
Processes :doc:`assimilated data <flags_criteria>` for the :doc:`amsua <families>` family globally. It generates 25 discrete PNG maps per cycle, representing **15-minute time bins** (from T-3h to T+3h). Each satellite or station ID is uniquely color-coded to track its trajectory across the global domain.

Interactive Session Features
----------------------------

Once the processing is complete, the module automatically generates a ``viewer_{family}.html`` file in your ``pathwork``. This interface provides a professional "Control Room" experience:

* **Temporal Navigation:** Use the interactive slider to scroll through the 25 bins and observe spatial displacement.
* **GIF Animation Mode:** A built-in player that cycles through frames at adjustable speeds (Slow, Normal, Fast).
* **Multi-Cycle Management:** A dynamic dropdown menu allows you to jump between different ``YYYYMMDDHH`` cycles without reloading the page.
* **Live Integrity Check:** The viewer displays real-time alerts if a specific time bin or image file is missing from the server.

**Web Viewer Interface:**

.. image:: _static/pikobszone_exp/sw/sw_2026021000_+000h_bin0.png
   :alt: Mapobs Interactive Session Viewer
   :align: center

*The session viewer uses relative path logic, ensuring it remains fully functional if the directory is moved to a different web server or documentation root.*

Live Documentation Links
------------------------

If you are browsing this documentation on the internal server, you can access the live interactive viewers directly:

.. note::
   The link below points to the internal static directory where the latest processed maps are hosted.


* **Interactive Session Viewer:** `Interactive Global Maps <`Interactive Global Maps <https://goc-dx-u3.science.gc.ca/~dlo001/sites8/pikobszone_exp/viewer_sw.html>`_ 


Support & Troubleshooting
-------------------------

If you encounter issues with the map generation, the binning logic, or the web interface functionality, please report them via the official channel:

👉 `Pikobs GitLab Issues <https://gitlab.science.gc.ca/dlo001/Pikobs/-/issues>`_ 

**When reporting an issue, please provide:**
1.  The exact ``python -c 'import pikobs...'`` command used.
2.  The ``family`` and ``date`` being processed.
3.  A screenshot of the web console if an image fails to load in the interactive session.
"""

#!/usr/bin/python3

import sqlite3
import pikobs
import re
import os
from  dask.distributed import Client
import numpy as np
import sqlite3
import os
import re
import sqlite3
from datetime import datetime, timedelta
import warnings
# Suppress all Shapely deprecation warnings
warnings.filterwarnings("ignore", ".*ShapelyDeprecationWarning.*")

# Now, import the libraries you're using
import shapely
import cartopy
def create_table_if_not_exists(cursor):
    """
    Create the 'serie_cardio' table if it does not already exist.

    Args:
        cursor (sqlite3.Cursor): Database cursor for executing SQL queries.
    """
    query = """
        CREATE TABLE IF NOT EXISTS moyenne (
            Nrej INTEGER,
            Nacc INTEGER,
            Nprofile INTEGER,
            DATE INTEGER,
            lat FLOAT,
            lon FLOAT,
            boite INTEGER,
            id_stn TEXT,
            varno INTEGER,
            vcoord FLOAT,   -- INTEGER FLOAT canal
            sumx FLOAT,
            sumy FLOAT,
            sumz FLOAT,
            sumStat FLOAT,
            sumx2 FLOAT,
            sumy2 FLOAT,
            sumz2 FLOAT,
            sumStat2 FLOAT,
            n INTEGER,
            flag INTEGER
        ); """
    cursor.execute(query)

def combine(pathfileout: str, filememory: str) -> None:
    """
    Combine multiple SQLite files into a single output file.

    This function copies records from the 'moyenne' table of a source SQLite file 
    (filememory) and inserts them into the 'moyenne' table of a destination/output SQLite file (pathfileout).
    
    Args:
        pathfileout (str): Path to the output SQLite file where combined data will be stored.
        filememory  (str): Path to the source SQLite file (can be memory or disk).
        
    Returns:
        None. The result is saved directly in the specified output file.
    """
    insert_sql = """
        INSERT INTO moyenne(
            Nrej, Nacc, Nprofile, DATE, lat, lon, boite, id_stn, varno, vcoord, sumx, sumy, sumz, 
            sumStat, sumx2, sumy2, sumz2, sumStat2, n, flag
        )
        SELECT 
            Nrej, Nacc, Nprofile, DATE, lat, lon, boite, id_stn, varno, vcoord, sumx, sumy, sumz, 
            sumStat, sumx2, sumy2, sumz2, sumStat2, n, flag
        FROM this_avg_db.moyenne
    """

    # Use 'with' statement to ensure connection is properly closed
    with sqlite3.connect(pathfileout, uri=True, isolation_level=None, timeout=9999) as conn:
        try:
            # Set performance-optimized PRAGMAs, provided your application allows for non-durable writes
            conn.execute("PRAGMA journal_mode=OFF;")
            conn.execute("PRAGMA synchronous=OFF;")

            # Ensure the table exists
            create_table_if_not_exists(conn)

            # Attach the source SQLite database
            conn.execute(f"ATTACH DATABASE '{filememory}' AS this_avg_db;")

            # Copy data
            conn.execute(insert_sql)
            conn.commit()

            # Detach the attached database
            conn.execute("DETACH DATABASE this_avg_db;")
        except sqlite3.Error as e:
            print(f"Error combining SQLite files: {e}")
            raise

def remove_combine(pathfileout, filememory):

    """ Creating a single sqlite file from multiple sqlite files
 
    args:
    ----------------

     pathfileout   : output averaging  sqlite file 
     filememory    : name of the sqlite file in memory to copy
   
    output:
    ----------------

    Nothing
     A sqlite file is made with all averaged volume scans
   
   """

    # write in output averaging  sqlite file
    conn_pathfileout = sqlite3.connect(pathfileout, uri=True, isolation_level=None, timeout=9999)

    conn_pathfileout.execute("""PRAGMA journal_mode=OFF;""")

    # off the journal
    create_table_if_not_exists( conn_pathfileout)
    conn_pathfileout.execute("""PRAGMA journal_mode=OFF;""")
    # SQLite continues without syncing as soon as it has handed data off to the operating system
    conn_pathfileout.execute("""PRAGMA synchronous=OFF;""")
    # Wait to read and write until the next process finishes
    # attach the sqlite file in memory for one PPI
    create_table_if_not_exists(conn_pathfileout)
    conn_pathfileout.execute(f"ATTACH DATABASE '{filememory}' AS this_avg_db;")
    order_sql = """ INSERT INTO moyenne( Nrej, Nacc, Nprofile, DATE, lat, lon, boite, id_stn, varno , vcoord, sumx, sumy, sumz, sumStat, sumx2,
                                         sumy2, sumz2, sumStat2, n, flag 
 

                )
                    SELECT Nrej, Nacc, Nprofile, DATE, lat, lon, boite, id_stn, varno , vcoord, sumx, sumy, sumz, sumStat, sumx2,
                                         sumy2, sumz2, sumStat2, n, flag 
 
 
                    FROM  this_avg_db.moyenne"""
    conn_pathfileout.execute(order_sql) 

    conn_pathfileout.commit()
    conn_pathfileout.execute(""" DETACH DATABASE this_avg_db """)

def remove_create_and_populate_moyenne_table(family, 
                                      new_db_filename,
                                      existing_db_filename,
                                      region_seleccionada,
                                      selected_flags, 
                                      FONCTION,
                                      boxsizex,
                                      boxsizey,
                                      varnos,
                                      channel,
                                      id_stn,
                                      interval):
    import sqlite3
    import re
    import pikobs  
    pattern = r'(\d{10})'
    match = re.search(pattern, existing_db_filename)
    if match:
        date = match.group(1)
    else:
        raise ValueError("No se encontraron 10 dígitos consecutivos en el nombre del archivo.")
    filename=f"file::memory:?cache=shared"

    new_db_conn = sqlite3.connect(filename, uri=True, isolation_level=None, timeout=9999999)
    new_db_cursor = new_db_conn.cursor()

    FAM, VCOORD, VCOCRIT, STATB, element, VCOTYP = pikobs.family(family)
    if varnos:
        element = ",".join(varnos)

    if channel == 'join' and id_stn == 'all':
        group_channel = '"join" as Chan,'
        group_id_stn = 'id_stn as id_stn,'
        group_id_stn_vcoord = 'group by 2, 3, 4, 5, id_stn'
    elif channel == 'all' and id_stn == 'join':
        group_channel = f'{VCOORD} as Chan,'
        group_id_stn = '"join" as id_stn,'
        group_id_stn_vcoord = f'group by 2, 3, 4, 5, {VCOORD}'
    elif channel == 'all' and id_stn == 'all':
        group_channel = f'{VCOORD} as Chan,'
        group_id_stn = 'id_stn as id_stn,'
        group_id_stn_vcoord = f'group by 2, 3, 4, 5, id_stn, {VCOORD}'
    elif channel == 'join' and id_stn == 'join':
        group_channel = '"join" as Chan,'
        group_id_stn = '"join" as id_stn,'
        group_id_stn_vcoord = 'group by 2, 3, 4, 5, date'

    LAT1, LAT2, LON1, LON2 = pikobs.regions(region_seleccionada)
    LATLONCRIT = pikobs.generate_latlon_criteria(LAT1, LAT2, LON1, LON2)
    flag_criteria = pikobs.flag_criteria(selected_flags)
    STNID = f"floor(360. / {boxsizex}) * floor(lat / {boxsizey}) + floor(MIN(179.99, lon) / {boxsizex})"
    LAT = f"floor(lat / {boxsizey}) * {boxsizey} + {boxsizey} / 2."
    LON = f"floor(MIN(179.99, lon) / {boxsizex}) * {boxsizex} + {boxsizex} / 2."

    new_db_cursor.execute("PRAGMA journal_mode = OFF;")
    new_db_cursor.execute("PRAGMA journal_mode = MEMORY;")
    new_db_cursor.execute("PRAGMA synchronous = OFF;")
    new_db_cursor.execute("PRAGMA foreign_keys = OFF;")
    new_db_cursor.execute(f"ATTACH DATABASE '{existing_db_filename}' AS db;")
    # Check if 'bias_corr' exists in the table schema
    bias_corr_exists = False
    new_db_cursor.execute("PRAGMA table_info('DATA')")
    columns = new_db_cursor.fetchall()
    interval_a=interval[0]
    interval_b=interval[1]

    if  interval_a==None and  interval_b==None:
          criteria_interval = ''
          layers='layer_all'
    else:
          criteria_interval = f'and vcoord >= {interval_a*100} AND vcoord <= {interval_b*100}'
          layers=f'layerfrom{interval_a}MPato{interval_b}MPa'


    for col in columns:
      if col[1] == 'bias_corr':  # Column name is at index 1
        bias_corr_exists = True
        break

    # Adjust the query based on the existence of 'bias_corr'
    bias_corr_sum = "sum(bias_corr)" if bias_corr_exists else "NULL"
    bias_corr_sq_sum = "sum(bias_corr * bias_corr)" if bias_corr_exists else "NULL"

    new_db_cursor.execute("""
        CREATE TABLE IF NOT EXISTS moyenne (
            Nrej INTEGER,
            Nacc INTEGER,
            Nprofile INTEGER,
            DATE INTEGER,
            lat FLOAT,
            lon FLOAT,
            boite INTEGER,
            id_stn TEXT,
            varno INTEGER,
            vcoord FLOAT,   -- INTEGER FLOAT canal
            sumx FLOAT,
            sumy FLOAT,
            sumz FLOAT,
            sumStat FLOAT,
            sumx2 FLOAT,
            sumy2 FLOAT,
            sumz2 FLOAT,
            sumStat2 FLOAT,
            n INTEGER,
            flag INTEGER
        ); """)
    qr = f"""
    INSERT INTO moyenne (
        DATE, lat, lon, boite, varno, vcoord, 
        sumx, sumy, sumz, sumStat, 
        sumx2, sumy2, sumz2, sumStat2, 
        n, Nrej, Nacc, Nprofile, id_stn, flag
    )
    SELECT
        {date},
        {LAT},
        {LON},
        {STNID},
        VARNO,
        {group_channel}
        sum(omp),
        sum(oma),
        sum(obsvalue), 
        {bias_corr_sum}, -- sum(bias_corr)
        sum(omp * omp),
        sum(oma * oma),
        sum(obsvalue * obsvalue),
        {bias_corr_sq_sum}, -- sum(bias_corr * bias_corr)
        COUNT(*),
        sum(flag & 512=512),
        sum(flag & 4096-4094),
        count(distinct id_obs),
        {group_id_stn}
        flag
    FROM
        db.header
    NATURAL JOIN
        db.DATA
    WHERE
        varno IN ({element}) 
        AND obsvalue IS NOT NULL
        {flag_criteria}
        {LATLONCRIT}
        {VCOCRIT}
        {criteria_interval}
    {group_id_stn_vcoord};
    """
    new_db_cursor.execute(qr)

    # Finalizar
    new_db_conn.commit()
    #new_db_conn.close()
    try:
         combine(new_db_filename, filename)
    except sqlite3.Error as error:
         print(f"Error while creating a single sqlite file:  {os.path.basename(filename)}", error)


import os
import re
import sqlite3
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import cartopy.feature as cfeature

# ==================================================
# 🛠️ CONFIGURACIÓN DE SALIDA
# ==================================================
# Cambia esta variable para que apunte a tu zona de publicación
TARGET_BASE_DIR = "/home/dlo001/sites8/pikobszone_exp"

def plot_bin_with_legend(df_bin, output_file, ref_date, bin_val):
    """Genera el mapa con Cartopy y leyenda de estaciones."""
    fig = plt.figure(figsize=(16, 9))
    ax = plt.axes(projection=ccrs.PlateCarree())
    
    # Capas geográficas
    ax.add_feature(cfeature.COASTLINE, linewidth=0.6)
    ax.add_feature(cfeature.LAND, facecolor='#fdfdfd')
    ax.add_feature(cfeature.BORDERS, linestyle=':', alpha=0.3)
    
    minutes_rel = bin_val * 15
    t_label = f"T{minutes_rel:+} min" if minutes_rel != 0 else "CYCLE T+0"

    if not df_bin.empty:
        # Agrupar por estación para asignar colores únicos
        unique_stns = sorted(df_bin['id_stn'].unique())
        colormap = plt.cm.get_cmap('tab20', len(unique_stns))
        
        for i, stn in enumerate(unique_stns):
            data = df_bin[df_bin['id_stn'] == stn]
            ax.scatter(
                data['lon'], data['lat'], 
                color=colormap(i), s=45, edgecolors='black', linewidth=0.4,
                label=f"ID: {stn}", transform=ccrs.PlateCarree()
            )
        
        # Leyenda lateral (ajusta columnas si hay muchas estaciones)
        ncol = 1 if len(unique_stns) < 12 else 2
        ax.legend(title="Stations", loc='center left', bbox_to_anchor=(1, 0.5), 
                  ncol=ncol, fontsize=8, frameon=True)
    else:
        ax.text(0.5, 0.5, f"EMPTY BIN ({t_label})", 
                ha='center', va='center', transform=ax.transAxes, fontsize=14, color='red')

    ax.set_title(f"Global Map | {t_label} | Ref: {ref_date}", fontsize=14, pad=15)
    ax.gridlines(draw_labels=True, linestyle='--', alpha=0.2)

    # Guardar con alta calidad
    plt.savefig(output_file, dpi=120, bbox_inches='tight')
    plt.close()
import os
import re

import os
import re
from typing import List, Tuple, Any, Dict, Optional, Union
def generate_web_viewer_from_dict(data_input: Any) -> None:
    """
    Generates an English interactive HTML viewer in the work_path.
    Detects all available dates by scanning the image folder.
    """
    # Handle list or dict input to avoid TypeError
    if isinstance(data_input, list):
        data_dict = data_input[0]
    else:
        data_dict = data_input

    family = data_dict['family']
    db_path = data_dict['db_new']
    region = data_dict['region']
    
    # 1. Determine the WORK_PATH and Image Folder
    split_pattern = f"/{family}/"
    if split_pattern in db_path:
        work_path = db_path.split(split_pattern)[0]
    else:
        work_path = os.path.dirname(os.path.dirname(db_path))
    
    img_folder = os.path.join(work_path, family)
    
    # 2. SCAN FOLDER: Extract all unique dates from existing .png files
    # Pattern expected: family_YYYYMMDDHH_offset_binX.png
    found_dates = set()
    if os.path.exists(img_folder):
        for f in os.listdir(img_folder):
            if f.endswith(".png"):
                match = re.search(r'_(\d{10})_', f)
                if match:
                    found_dates.add(match.group(1))
    
    all_dates = sorted(list(found_dates))
    
    # Fallback to DB path dates if folder scan fails
    if not all_dates:
        all_dates = sorted(list(set(re.findall(r'(\d{10})', db_path))))
        if not all_dates: 
            all_dates = ["0000000000"]

    output_path = os.path.join(work_path, f"viewer_{family}.html")
    date_options = "".join([f'<option value="{d}">{d}</option>' for d in all_dates])

    # 3. HTML CONTENT (English)
    html_content = f"""
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Pikobs Web Viewer | {family.upper()}</title>
    <style>
        body {{ font-family: 'Segoe UI', sans-serif; background: #121212; color: #eee; text-align: center; padding: 15px; }}
        .card {{ max-width: 1150px; margin: auto; background: #1e1e1e; padding: 20px; border-radius: 12px; border: 1px solid #333; box-shadow: 0 10px 40px #000; }}
        .header {{ display: flex; justify-content: space-between; align-items: center; border-bottom: 2px solid #3498db; padding-bottom: 10px; margin-bottom: 15px; }}
        .map-area {{ background: #000; border-radius: 8px; margin: 15px 0; min-height: 600px; display: flex; align-items: center; justify-content: center; border: 1px solid #333; }}
        img {{ max-width: 100%; height: auto; }}
        .controls {{ background: #252525; padding: 20px; border-radius: 10px; }}
        .info {{ font-size: 1.3rem; color: #3498db; font-weight: bold; margin-bottom: 10px; }}
        button {{ background: #3498db; color: white; border: none; padding: 10px 25px; border-radius: 5px; cursor: pointer; font-weight: bold; margin: 0 5px; }}
        button:hover {{ background: #2980b9; }}
        .btn-stop {{ background: #e74c3c !important; }}
        select {{ background: #333; color: white; padding: 8px; border-radius: 5px; border: 1px solid #555; }}
        input[type=range] {{ width: 85%; cursor: pointer; }}
    </style>
</head>
<body>
<div class="card">
    <div class="header">
        <h2 style="margin:0;">🛰️ {family.upper()} Interactive Monitor</h2>
        <div>
            <span>Select Date: </span>
            <select id="date-select">{date_options}</select>
        </div>
    </div>
    <div class="info" id="status">Initializing...</div>
    <div class="map-area"><img id="main-img" src="" alt="Satellite Map"></div>
    <div class="controls">
        <input type="range" id="slider" min="-12" max="12" value="0">
        <br><br>
        <button id="prev">⬅️ Prev</button>
        <button id="play" style="width:160px;">▶️ Play Animation</button>
        <button id="next">Next ➡️</button>
        <span style="margin-left:25px;">Speed: </span>
        <select id="speed">
            <option value="600">Slow</option>
            <option value="300" selected>Normal</option>
            <option value="100">Fast</option>
        </select>
    </div>
    <p style="color:#666; font-size:0.8rem; margin-top:15px;">Region: {region} | Path: {work_path}</p>
</div>
<script>
    const cfg = {{ family: "{family}", path: "{family}/" }};
    const img = document.getElementById('main-img');
    const slider = document.getElementById('slider');
    const dateSel = document.getElementById('date-select');
    const status = document.getElementById('status');
    const playBtn = document.getElementById('play');
    let timer = null;

    function update() {{
        const date = dateSel.value;
        const bin = parseInt(slider.value);
        const h = bin * 0.25;
        const hStr = ((h >= 0 ? "+" : "") + h.toFixed(2) + "h").replace(".", "");
        
        const src = `${{cfg.path}}${{cfg.family}}_${{date}}_${{hStr}}_bin${{bin}}.png`;
        img.src = src;
        
        const m = bin * 15;
        status.innerText = `Cycle: ${{date}} | Offset: ${{m >= 0 ? '+' : ''}}${{m}} min (Bin ${{bin}})`;
        
        img.onerror = () => {{ 
            status.innerText = "⚠️ Missing Frame: " + src.split('/').pop(); 
            status.style.color = "#e67e22"; 
        }};
        img.onload = () => {{ status.style.color = "#3498db"; }};
    }}

    slider.oninput = update;
    dateSel.onchange = update;
    const nextFrame = () => {{ 
        slider.value = (parseInt(slider.value) >= 12) ? -12 : parseInt(slider.value) + 1; 
        update(); 
    }};
    document.getElementById('next').onclick = nextFrame;
    document.getElementById('prev').onclick = () => {{ 
        slider.value = (parseInt(slider.value) <= -12) ? 12 : parseInt(slider.value) - 1; 
        update(); 
    }};
    
    playBtn.onclick = () => {{
        if (timer) {{ 
            clearInterval(timer); timer = null; 
            playBtn.innerText = "▶️ Play Animation"; playBtn.classList.remove('btn-stop'); 
        }} else {{ 
            playBtn.innerText = "⏹️ Stop"; playBtn.classList.add('btn-stop'); 
            timer = setInterval(nextFrame, parseInt(document.getElementById('speed').value)); 
        }}
    }};
    update();
</script>
</body>
</html>
    """
    with open(output_path, "w", encoding="utf-8") as f:
        f.write(html_content)
    print(f"✅ SUCCESS: Web Viewer generated at {output_path}")
import os
import re
from typing import List, Tuple, Any, Dict, Optional, Union
def create_and_populate_moyenne_table(
    family, new_db_filename, existing_db_filename, selected_region, 
    selected_flags, fonction, boxsizex, boxsizey, varnos, 
    channel, id_stn, interval, 
    output_plots_dir="/home/dlo001/sites8/pikobszone_exp"
) -> None:
    
    import pikobs

    # 1. FECHA Y DIRECTORIO
    date_match = re.search(r'(\d{10})', existing_db_filename)
    date_str = date_match.group(1) if date_match else "0000000000"
    target_path = os.path.join(output_plots_dir, family)
    os.makedirs(target_path, exist_ok=True)

    # 2. SQL
    FAM, VCOORD, VCOCRIT, STATB, element_def, VCOTYP = pikobs.family(family)
    flag_sql = pikobs.flag_criteria(selected_flags)
    LAT1, LAT2, LON1, LON2 = pikobs.regions(selected_region)
    LATLONCRIT = pikobs.generate_latlon_criteria(LAT1, LAT2, LON1, LON2)
    element_sql = ",".join(map(str, varnos)) if varnos else element_def

    with sqlite3.connect(":memory:") as mem_conn:
        query = f"""
            SELECT h.id_stn, h.lat, h.lon, h.date, h.time, d.varno
            FROM db.header h JOIN db.data d ON h.id_obs = d.id_obs
            WHERE d.varno IN ({element_sql}) AND d.obsvalue IS NOT NULL {flag_sql} {LATLONCRIT}
        """
        mem_conn.execute(f"ATTACH DATABASE '{existing_db_filename}' AS db;")
        df = pd.read_sql(query, mem_conn)

    # 3. LÓGICA DE BINS BLINDADA (±180 minutos)
    if not df.empty:
        df.columns = [c.lower() for c in df.columns]
        
        # Convertir a datetime real
        ts = df['date'].astype(str).str.strip() + df['time'].astype(str).str.strip().str.zfill(6)
        df['dt'] = pd.to_datetime(ts, format="%Y%m%d%H%M%S", errors='coerce')
        ref_dt = pd.to_datetime(date_str, format="%Y%m%d%H")
        
        # Diferencia en MINUTOS totales
        df['diff_min'] = (df['dt'] - ref_dt).dt.total_seconds() / 60.0
        
        # Filtrar ventana estricta de ±180 min (3 horas)
        df = df[(df['diff_min'] >= -180) & (df['diff_min'] <= 180)].copy()
        
        # CÁLCULO DEL BIN: 
        # Dividimos por 15 y redondeamos al entero más cercano. 
        # Esto nos da un rango de -12 a +12 (25 posibles bins)
        df['tbin'] = np.round(df['diff_min'] / 15.0).astype(int)

    # 4. GENERACIÓN DE LOS 24 PLOTS (Se recorren todos los bins posibles)
    bins_a_generar = range(-12, 13) # De -3h a +3h en pasos de 15min

    for b in bins_a_generar:
        df_bin = df[df['tbin'] == b] if (not df.empty and 'tbin' in df.columns) else pd.DataFrame()
        
        # Formatear offset para el nombre del fichero (ej: -3.0h -> -300h)
        offset_h = b * 0.25
        offset_str = f"{offset_h:+.2f}h".replace(".", "")
        
        # NOMBRE: family_date_offset_binX.png
        filename = f"{family}_{date_str}_{offset_str}_bin{b}.png"
        full_path = os.path.join(target_path, filename)

        # CONFIGURACIÓN DEL MAPA MUNDIAL
        fig = plt.figure(figsize=(15, 8))
        ax = plt.axes(projection=ccrs.PlateCarree())
        ax.set_global() # Ver la Tierra entera
        
        ax.add_feature(cfeature.COASTLINE, linewidth=0.5)
        ax.add_feature(cfeature.LAND, facecolor='#f4f4f4')
        ax.add_feature(cfeature.OCEAN, facecolor='#eafaf1')
        ax.add_feature(cfeature.BORDERS, linestyle=':', alpha=0.3)

        if not df_bin.empty:
            stns = sorted(df_bin['id_stn'].unique())
            # Paleta para estaciones
            cmap = plt.cm.get_cmap('tab20', len(stns))
            
            for i, s in enumerate(stns):
                d_stn = df_bin[df_bin['id_stn'] == s]
                ax.scatter(d_stn['lon'], d_stn['lat'], color=cmap(i), 
                           s=40, edgecolors='black', linewidth=0.3, label=str(s),
                           transform=ccrs.PlateCarree())
            
            # Leyenda si hay estaciones
            if 0 < len(stns) < 40:
                ax.legend(title="STN ID", loc='center left', bbox_to_anchor=(1, 0.5), 
                          ncol=2 if len(stns) > 20 else 1, fontsize=7)
        
        ax.set_title(f"Global Map | Bin {b} ({offset_h:+.2f}h)\nCycle: {date_str}", fontsize=12)
        ax.gridlines(draw_labels=True, linestyle='--', alpha=0.2)

        plt.savefig(full_path, dpi=100, bbox_inches='tight')
        plt.close()
        print(f"✅ Creado: {filename}")

    print(f"\n🎉 24 Mapas generados en {target_path}")
from datetime import datetime, timedelta
from typing import List, Dict, Any, Tuple

def create_data_list(
    date_start: str,
    date_end: str,
    families: List[str],
    input_paths: List[str],
    names: List[str],
    work_path: str,
    box_size_x: float,
    box_size_y: float,
    function: str,
    flag_criteria: str,
    regions: List[Any],       # adjust type if you have a region class
    varnos: List[str],
    intervals: List[Tuple[Any, Any]],
) -> List[Dict[str, Any]]:
    """
    Builds a list of dictionaries containing all configuration info 
    for data processing at 6-hour intervals, for multiple families, 
    regions, input files, and intervals.

    Args:
        date_start (str): Start date string in format 'YYYYMMDDHH'.
        date_end (str): End date string in format 'YYYYMMDDHH'.
        families (List[str]): List of family names/codes.
        input_paths (List[str]): List of input file path prefixes.
        names (List[str]): List of dataset names.
        work_path (str): Path where output databases should be created.
        box_size_x (float): Size of the box in longitude.
        box_size_y (float): Size of the box in latitude.
        function (str): Function being applied (name/purpose).
        flag_criteria (str): Flag criteria (as a code or query substring).
        regions (List[Any]): List of region specifications.
        varnos (List[str]): List of variable numbers/codes.
        intervals (List[Tuple[Any, Any]]): List of interval tuples (min, max).

    Returns:
        List[Dict[str, Any]]: List of configuration dictionaries.
    """
    data_list = []

    dt_start = datetime.strptime(date_start, '%Y%m%d%H')
    dt_end = datetime.strptime(date_end, '%Y%m%d%H')
    delta = timedelta(hours=6)
    current_date = dt_start

    while current_date <= dt_end:
        formatted_date = current_date.strftime('%Y%m%d%H')
        # Nested loops over all combinations
        for family in families:
            for region in regions:
                for name, input_path in zip(names, input_paths):
                    for interval in intervals:
                        interval_a, interval_b = interval
                        if interval_a is None and interval_b is None:
                            layers = 'layer_all'
                        else:
                            layers = f'layer_from{interval_a}MPato{interval_b}MPa'

                        filename = f'{formatted_date}_{family}'
                        filein = f'{input_path}/{filename}'
                        db_new = (
                            f'{work_path}/{family}/scatter_{layers}_{name}_{region}_'
                            f'{date_start}_{date_end}_bx{box_size_x}_by{box_size_y}_'
                            f'{flag_criteria}_{family}.db'
                        )

                        data_dict = {
                            'family': family,
                            'filein': filein,
                            'db_new': db_new,
                            'region': region,
                            'flag_criteria': flag_criteria,
                            'function': function,
                            'boxsizex': box_size_x,
                            'boxsizey': box_size_y,
                            'varnos': varnos,
                            'interval': interval
                        }
                        data_list.append(data_dict)
        current_date += delta

    return data_list   
def remove_create_data_list(datestart1, 
                     dateend1,
                     families,
                     pathins, 
                     names, 
                     pathwork,
                     boxsizex, 
                     boxsizey, 
                     fonction, 
                     flag_criteria, 
                     regions, 
                     varnos,
                     intervales):
       data_list = []

       # Convert datestart and dateend to datetime objects
       datestart = datetime.strptime(datestart1, '%Y%m%d%H')
       dateend = datetime.strptime(dateend1, '%Y%m%d%H')

       # Initialize the current_date to datestart
       current_date = datestart

       # Define a timedelta of 6 hours
       delta = timedelta(hours=6)
   
       # Iterate through the date range in 6-hour intervals
       while current_date <= dateend:
               for family in families:
                for region in regions:
                 for name, pathin in zip(names,pathins):
                  for interval in intervales:
               #   for pathin in pathins:
                    interval_a=interval[0]
                    interval_b=interval[1]
                    if  interval_a==None and  interval_b==None:
                       criteria_interval = ''
                       layers='layer_all'
                    else:
                       criteria_interval = f' and  {interval_a*100} <= vcoord <= {interval_b*100}'
                       layers=f'layer_from{interval_a}MPato{interval_b}MPa'
                    # Format the current date as a string
                    formatted_date = current_date.strftime('%Y%m%d%H')
   
                    # Build the file name using the date and family
                    filename = f'{formatted_date}_{family}'
                    # Create a new dictionary and append it to the list
                    data_dict = {
                      'family': family,
                      'filein': f'{pathin}/{filename}',
                      'db_new': f'{pathwork}/{family}/scatter_{layers}_{name}_{region}_{datestart1}_{dateend1}_bx{boxsizex}_by{boxsizey}_{flag_criteria}_{family}.db',
                      'region': region,
                      'flag_criteria': flag_criteria,
                      'fonction': fonction,
                      'boxsizex': boxsizex,
                      'boxsizey': boxsizey,
                      'varnos':varnos,
                      'interval':interval
                    }
                    data_list.append(data_dict)
   
                    # Update the current_date in the loop by adding 6 hours
               current_date += delta
   
       return data_list
   
import sqlite3
import numpy as np

def get_id_stns(cursor, id_stn):
    """Obtiene la lista de id_stn desde la base de datos."""
    if id_stn == 'all':
        query = "SELECT DISTINCT id_stn FROM moyenne;"
        cursor.execute(query)
        return np.array([item[0] for item in cursor.fetchall()])
    return ['join']

def fetch_vcoords(cursor, criter):
    """Obtiene las coordenadas verticales y números de variable según el criterio dado."""
    query = f"SELECT DISTINCT 'all', varno FROM moyenne {criter} ORDER BY vcoord ASC;"
    cursor.execute(query)
    return cursor.fetchall()

def fetch_channels_varno(cursor):
    """Obtiene los números de variable de todos los canales."""
    query = "SELECT DISTINCT varno FROM moyenne ORDER BY vcoord ASC;"
    cursor.execute(query)
    return [item[0] for item in cursor.fetchall()]

def remove_create_data_list_plot(datestart1, dateend1, families, namein, pathwork, boxsizex, boxsizey,
                          fonctions, flag_criteria, regions, id_stn, channel, projs,intervales):
    data_list_plot = []
    for interval in intervales:
      for family in families:
       for region in regions: 
        for fonction in fonctions:
         for proj in projs:
          interval_a = interval[0]
          interval_b = interval[1]
          
          if  interval_a==None and  interval_b==None:
            criteria_interval = ''
            layers='layer_all'
          else:
            criteria_interval = f' and  {interval_a*100} <= vcoord <= {interval_b*100}'
            layers=f'layer_from{interval_a}MPato{interval_b}MPa'
          
          fileset = [f'{pathwork}/{family}/scatter_{layers}_{namein[0]}_{region}_{datestart1}_{dateend1}_bx{boxsizex}_by{boxsizey}_{flag_criteria}_{family}.db']
          nameset = [namein[0]]
      
          if len(namein) > 1:
              fileb = f'{pathwork}/{family}/scatter_{layers}_{namein[1]}_{region}_{datestart1}_{dateend1}_bx{boxsizex}_by{boxsizey}_{flag_criteria}_{family}.db'
              fileset.append(fileb)
              nameset.append(namein[1])
          with sqlite3.connect(fileset[0]) as conn:
              cursor = conn.cursor()
              id_stns = get_id_stns(cursor, id_stn)
      
              for idstn in id_stns:
                  criter = '' if id_stn == 'join' else f'WHERE id_stn = "{idstn}"'
                  
                  if channel == 'all':
      
                      vcoords = fetch_vcoords(cursor, criter)
                      for vcoord, varno in vcoords:
                          data_list_plot.append({
                              'id_stn': idstn,
                              'vcoord': vcoord,
                              'files_in': fileset,
                              'varno': varno,
                              'region': region,
                              'fonction':fonction,
                              'proj':proj,
                              'interval':interval,
                              'family':family

  
                          })
                  else:
                      channels_varno = fetch_channels_varno(cursor)
                      #print (channels_varno)
                      for  varno in channels_varno:
                     # if channels_varno:
                          data_list_plot.append({
                              'id_stn': idstn,
                              'vcoord': 'join',
                              'files_in': fileset,
                              'varno': varno,
                              'region': region,
                              'fonction':fonction,
                              'proj':proj ,
                              'interval':interval,
                              'family':family


  
                          })
    return data_list_plot
def create_data_list_plot(
    date_start: str,
    date_end: str,
    families: List[str],
    namein: List[str],
    work_path: str,
    box_size_x: float,
    box_size_y: float,
    functions: List[str],
    flag_criteria: str,
    regions: List[Any],
    id_stn: str,
    channel: str,
    projections: List[str],
    intervals: List[Tuple[Any, Any]],
) -> List[Dict[str, Any]]:
    """
    Generates a list of plotting dataset configurations for different intervals,
    families, regions, functions, channels, projections, and station IDs.

    Args:
        date_start (str): Starting date as 'YYYYMMDDHH'.
        date_end (str): Ending date as 'YYYYMMDDHH'.
        families (List[str]): List of family names/codes.
        namein (List[str]): Input dataset names (should be 1 or 2).
        work_path (str): Path prefix where database files are located.
        box_size_x (float): Box size in longitude.
        box_size_y (float): Box size in latitude.
        functions (List[str]): List of function names or codes.
        flag_criteria (str): Criteria as string for filtering.
        regions (List[Any]): List of region specifications.
        id_stn (str): 'all', 'join', or specific station ID.
        channel (str): 'all' or 'join'.
        projections (List[str]): List of projections to produce.
        intervals (List[Tuple[Any, Any]]): List of interval tuples (min, max).

    Returns:
        List[Dict[str, Any]]: List of plot dataset configuration dictionaries.
    """
    data_list_plot = []

    for interval in intervals:
        interval_a, interval_b = interval
        if interval_a is None and interval_b is None:
            layers = 'layer_all'
        else:
            layers = f'layer_from{interval_a}MPato{interval_b}MPa'

        for family in families:
            for region in regions:
                for function in functions:
                    for proj in projections:
                        # Build list of database files for this config
                        fileset = [
                            f'{work_path}/{family}/scatter_{layers}_{namein[0]}_{region}_{date_start}_{date_end}_bx{box_size_x}_by{box_size_y}_{flag_criteria}_{family}.db'
                        ]
                        nameset = [namein[0]]
                        if len(namein) > 1:
                            file_b = f'{work_path}/{family}/scatter_{layers}_{namein[1]}_{region}_{date_start}_{date_end}_bx{box_size_x}_by{box_size_y}_{flag_criteria}_{family}.db'
                            fileset.append(file_b)
                            nameset.append(namein[1])

                        # Open the first file in the set and process station IDs and vcoords
                        try:
                            with sqlite3.connect(fileset[0]) as conn:
                                cursor = conn.cursor()
                                id_stns = get_id_stns(cursor, id_stn)    # Returns List[str]

                                for idstn in id_stns:
                                    where_clause = '' if id_stn == 'join' else f'WHERE id_stn = "{idstn}"'
                                    if channel == 'all':
                                        vcoords = fetch_vcoords(cursor, where_clause)  # Returns List[Tuple[Any, int]]
                                        for vcoord, varno in vcoords:
                                            data_list_plot.append({
                                                'id_stn': idstn,
                                                'vcoord': vcoord,
                                                'files_in': fileset,
                                                'varno': varno,
                                                'region': region,
                                                'function': function,
                                                'proj': proj,
                                                'interval': interval,
                                                'family': family
                                            })
                                    else:  # channel == 'join' or other
                                        channels_varnos = fetch_channels_varno(cursor)  # Returns List[int]
                                        for varno in channels_varnos:
                                            data_list_plot.append({
                                                'id_stn': idstn,
                                                'vcoord': 'join',
                                                'files_in': fileset,
                                                'varno': varno,
                                                'region': region,
                                                'function': function,
                                                'proj': proj,
                                                'interval': interval,
                                                'family': family
                                            })
                        except sqlite3.Error as err:
                            print(f"Warning: Could not open or query {fileset[0]}: {err}")
    return data_list_plot
 
def make_scatter(files_in,
                 names_in,  
                 pathwork, 
                 datestart,
                 dateend,
                 regions, 
                 families, 
                 flag_criteria, 
                 fonctions, 
                 varnos,
                 boxsizex, 
                 boxsizey, 
                 projs, # Proj=='OrthoN'// Proj=='OrthoS'// Proj=='robinson' // Proj=='Europe' // Proj=='Canada' // Proj=='AmeriqueNord' // Proj=='Npolar' //  Proj=='Spolar' // Proj == 'reg'
                 mode,
                 Points,
                 id_stn,
                 channel,
                 n_cpu,
                 intervales):
   
        """
       Perform scatter plot generation based on input parameters.
   
       Args:

        files_in (list): List of input file paths.

        names_in (list): List of input file names.

        pathwork (str): Working directory.

        datestart (str): Start date in YYYYMMDDHH format.

        dateend (str): End date in YYYYMMDDHH format.

        region (str): Region parameter description.

        family (str): Family parameter description.

        flag_criteria (str): Flags criteria.

        fonction (str): Function parameter description.

        boxsizex (int): Box size in X direction.

        boxsizey (int): Box size in Y direction.

        Proj (str or list): Projection type ('cyl', 'OrthoN', 'OrthoS', etc.).

        mode (str): Mode parameter description.

        Points (str): Points parameter description.

        id_stn (str): id_stn parameter description.

        channel (str): Channel parameter description.

        n_cpu (int): Number of CPUs to use.
   
       Returns:
       
         None
        """
        for family in families:
              pikobs.delete_create_folder(pathwork, family)
        
        print (varnos)
        data_list = create_data_list(datestart,
                                       dateend, 
                                       families, 
                                       files_in,
                                       names_in,
                                       pathwork,
                                       boxsizex,
                                       boxsizey, 
                                       fonctions, 
                                       flag_criteria, 
                                       regions,
                                       varnos,
                                       intervales)
                                       
       # exit()                             
        import time
        import dask
        t0 = time.time()
      #  n_cpu=1
        if n_cpu==1:
          print (f'in Serie files: {len(data_list)} used in calculating statistics for {names_in}')
          for  data_ in data_list:  
               create_and_populate_moyenne_table(data_['family'], 
                                                 data_['db_new'], 
                                                 data_['filein'],
                                                 data_['region'],
                                                 data_['flag_criteria'],
                                                 data_['fonction'],
                                                 data_['boxsizex'],
                                                 data_['boxsizey'],
                                                 data_['varnos'],
                                                 channel,
                                                 id_stn,
                                                 data_['interval'])
               
       
       
       
       
        else:
           print (f'in Paralle number of files: {len(data_list)} used in calculating statistics for {len(data_list)}  {names_in} ')
           with dask.distributed.Client(processes=True, threads_per_worker=1, 
                                              n_workers=n_cpu, 
                                              silence_logs=40) as client:
               delayed_funcs = [dask.delayed(create_and_populate_moyenne_table)(data_['family'], 
                                                 data_['db_new'], 
                                                 data_['filein'],
                                                 data_['region'],
                                                 data_['flag_criteria'],
                                                 data_['function'],
                                                 data_['boxsizex'],
                                                 data_['boxsizey'],
                                                 data_['varnos'],
                                                 channel,
                                                 id_stn,
                                                 data_['interval'])for data_ in data_list]
              # print ('close0')
               results = dask.compute(*delayed_funcs)
               client.close()

        tn= time.time()
        print ('Total time:',time.time() - t0 )  
        print (f'check: {pathwork}') 
        print (data_list)
        generate_web_viewer_from_dict(data_list) 
    
import argparse
import re
import argparse
import re

def parse_intervals(text):
    """Parses a string of intervals formatted as '[(,),(a,b)]' into a list of tuples."""

    # Expresión regular corregida para aceptar paréntesis vacíos y números
    pattern = r'\(\s*(\d*)\s*,\s*(\d*)\s*\)'  
    matches = re.findall(pattern, text)


    result = []
    for a, b in matches:
        val_a = int(a) if a.isdigit() else None
        val_b = int(b) if b.isdigit() else None
        result.append((val_a, val_b))

    return result

def arg_call():

    import argparse
    import sys
    parser = argparse.ArgumentParser()
    parser.add_argument('--path_control_files', default='undefined', type=str, help="Directory where input sqlite files are located")
    parser.add_argument('--control_name', default='undefined', type=str, help="Directory where input sqlite files are located")
    parser.add_argument('--path_experience_files', default='undefined', type=str, help="Directory where input sqlite files are located")
    parser.add_argument('--experience_name', default='undefined', type=str, help="Directory where input sqlite files are located")
  

    
    parser.add_argument('--pathwork', default='undefined', type=str, help="Working directory")
    parser.add_argument('--datestart', default='undefined', type=str, help="Start date")
    parser.add_argument('--dateend', default='undefined', type=str, help="End date")
    parser.add_argument('--region', nargs="+", default='undefined', type=str, help="Region")
    parser.add_argument('--family', nargs="+", default='undefined', type=str, help="Family")
    parser.add_argument('--flags_criteria', default='undefined', type=str, help="Flags criteria")
    parser.add_argument('--fonction', nargs="+", default='omp', type=str, help="Function") 
    parser.add_argument('--varnos', nargs="+", default='undefined', type=str, help="Function")
    parser.add_argument('--boxsizex', default=2, type=int, help="Box size in X direction")
    parser.add_argument('--boxsizey', default=2, type=int, help="Box size in Y direction")
    parser.add_argument('--projection', nargs="+", default='cyl', type=str, help="Projection type (cyl, OrthoN, OrthoS, robinson, Europe, Canada, AmeriqueNord, Npolar, Spolar, reg)")
    parser.add_argument('--mode', default='SIGMA', type=str, help="Mode")
    parser.add_argument('--Points', default='OFF', type=str, help="Points")
    parser.add_argument('--id_stn', default='one_per_plot', type=str, help="id_stn") 
    parser.add_argument('--channel', default='one_per_plot', type=str, help="channel")
    parser.add_argument('--n_cpus', default=1, type=int, help="Number of CPUs")
    parser.add_argument(
    "--layer",
    type=str,
    nargs="?",
    default="[(,)]",  # Valor por defecto
    help="Lista de intervalos, ejemplo: '[(,),(700,900)]'"
)
    
    args = parser.parse_args()
    for arg in vars(args):
       print (f'--{arg} {getattr(args, arg)}')
    # Check if each argument is 'undefined'
    if args.path_control_files == 'undefined':
        files_in = [args.path_experience_files]
        names_in = [args.experience_name]
    else:    
        if args.path_experience_files == 'undefined':
            raise ValueError('You must specify --path_experience_files')
        if args.experience_name == 'undefined':
            raise ValueError('You must specify --experience_name')
        else:

            files_in = [args.path_control_files, args.path_experience_files]
            names_in = [args.control_name, args.experience_name] 
    intervals = parse_intervals(args.layer)
    args = parser.parse_args()


    if args.varnos == 'undefined':
        args.varnos = []
    if args.pathwork == 'undefined':
        raise ValueError('You must specify --pathwork')
    if args.datestart == 'undefined':
        raise ValueError('You must specify --datestart')
    if args.dateend == 'undefined':
        raise ValueError('You must specify --dateend')
    if args.region == 'undefined':
        raise ValueError('You must specify --region')
    if args.family == 'undefined':
        raise ValueError('You must specify --family')
    if args.flags_criteria == 'undefined':
        raise ValueError('You must specify --flags_criteria')
    if args.fonction == 'undefined':
        raise ValueError('You must specify --fonction')
    if args.boxsizex == 'undefined':
        raise ValueError('You must specify --boxsizex')
    if args.boxsizey == 'undefined':
        raise ValueError('You must specify --boxsizey')


    # Comment
    # Proj='cyl' // Proj=='OrthoN'// Proj=='OrthoS'// Proj=='robinson' // Proj=='Europe' // Proj=='Canada' // Proj=='AmeriqueNord' // Proj=='Npolar' //  Proj=='Spolar' // Proj == 'reg'
  

    #print("in")
    # Call your function with the arguments
    sys.exit(make_scatter(files_in,
                          names_in,    
                          args.pathwork,
                          args.datestart,
                          args.dateend,
                          args.region,
                          args.family,
                          args.flags_criteria,
                          args.fonction,
                          args.varnos,
                          args.boxsizex,
                          args.boxsizey,
                          args.projection,
                          args.mode,
                          args.Points,
                          args.id_stn,
                          args.channel,
                          args.n_cpus,
                          intervals))
