import os, sys, warnings
warnings.simplefilter('ignore')
from dateutil.relativedelta import relativedelta
import calendar

import numpy as np
import pandas as pd

from .modules import moving_direction, conv_second
from .modules import great_circle_distance_numba as gcd
from .circle_overlap import circle, circle_overlap


class Track():
    """
    Tracking program for outputs of colindex2. 
    Core of the tracking algorithm is to search nearest candidate 
    toward adjacent future and past tiemsteps.
    See puplication for details of the algorithm.

    ----

    Parameters:
    -----------

    **ty** : ``str``, defoult ``'both'``
        Type of depression.

        - ``'L'`` for only lows (cutoff lows and troughs).
        - ``'H'`` for only highs (cutoff highs and ridges).
        - ``'both'`` for both.


    **times** : ``pd.DatetimeIndex``
        Timesteps for tracking using ``pandas.DatetimeIndex``,
        which can be generated by ``pandas.date_range()``.


    **lev** : ``int``, default ``0``
        The level of vertical axis to be tracked.


    **odir** : ``str``, default ``'./d01'``
        Path to outputs from colindex2.

        .. code-block:: text

            current_dir/
            └── d01/
                ├── V/
                |   └── V-{ty}-{yyyymmddhhTT}-{llll}.csv
                ├── Vt/
                |   └── V-{ty}-{yyyymmddhhTT}-{llll}.csv
                └── Vtc/
                    └── V-{ty}-{yyyymmddhhTT}-{llll}.csv

        In this process, csv files in ``V`` directory will be read,
        intermediate csv files will be stored in ``Vt``,
        finally, the result csv files will be stored in ``Vtc``.


    **long_term** : ``bool``, default ``False``
        For long term analyses, especially for longer than a year. 
        If ``True``, tracking ID will be labeled from 1 in every 1st Jan and counted up till 31th Dec.
        If ``False``, tracking ID will be continuously counted up during the analysis period, and massive memory usage may occur.


    **operational** : ``bool``, default ``False``
        Note! this feature is experimental.
        If True, resume tracking with tracking data (files in ``{odir}/Vt``) produced in advance,
        so that you can restart tracking from whenever you like.
        Make sure times must NOT include the end-of-timestep flag, i.e., ``MERGE=-3``.
        If there are data ending at ``2020-12-31 18`` with a 6-hour timestep,
        the operational tracking must start before ``2020-12-31 12``.


    **tlimit** : ``float``, default ``150.``
        Traveling speed limit to prevent wrong connections by large depressions [km/h].
        
        Default 150 km/h is 900 km in 6 hour. 


    **DU_thres** : ``int``, default ``0``
        Threshold for noise removal with respect to duration (life time) [hour].


    **MAXSo_thres** : ``float``, default ``0.``
        Threshold for noise removal with respect to maximum intensity in life [m/100km].


    **ALLDIST_thres** : ``float``, default ``0.``
        Threshold for noise removal with respect to total moving distance in life [km].

    **only_count** : ``bool``, default ``False``
        If ``True``, skip main tracking process.

    ----

    Examples:
    ---------

    Set date range to be tracked. ::

        >>> import pandas as pd
        >>> times = pd.date_range('2010-01-01 00', '2010-01-31 18', freq='6h')

    Import and execute ``Track`` with ``times`` and ``level`` (and ``odir=odir``).
    In this case, the level is 300 hPa and ``./d01`` directory will be used. ::

        >>> from colindex2 import Track
        >>> Track(times, 300)

    That is the basic use of Tracker. Following is for some special cases.

    For long term analysis beyond a year ::

        >>> times = pd.date_range('1959-01-01 00', '2020-12-01 18', freq='6h')
        >>> Track(times, 300, long_term=True)

    If you interested only in lows or highs, set ``ty='L'`` or ``ty='H'``.
    By default, both types will be tracked. ::

        >>> Track(times, 300, ty='L')

    An example using multiprocessing. ::

        >>> import multiprocessing as mp
        >>> from colindex import Tracker
        >>> import pandas as pd
        ...
        >>> def main(args):
        ...     times = pd.date_range('2018-11-01 00', '2019-03-31 18', freq='6h')
        ...     Track(times, args[1], ty=args[0])
        ...
        >>> if __name__ == '__main__':
        ...     # double-nested args for types and levels
        ...     args = [[ty, lev] for ty in ['L', 'H'] for lev in [500, 300, 200]]
        ...     with mp.Pool(len(args)) as p:
        ...         p.map(main, args)
    """

    def __init__(self, times, lev, ty='both',
                 odir='./d01', tlimit=150.,# normalized_speed=133.33,
                 penalty=0,
                 long_term=False, operational=False, 
                 DU_thres=0, MAXSo_thres=0., ALLDIST_thres=0.,
                 only_count=False,
                 ):

        self.odir = odir
        self.lev = lev
        self.ty = ty
        self.penalty = penalty
        self.times = times
        self.operational = operational
        self.long_term = long_term
        self.DU_thres = DU_thres
        self.MAXSo_thres = MAXSo_thres
        self.ALLDIST_thres = ALLDIST_thres

        # sub params for counting when long_term=True
        # consider longest duration of depression
        self.addmonth = 3

        # prevent ID overflow for long term analysis
        if self.long_term:
            self.yearly_ID_update = True  # for track
            self.read_csv_duration = 'monthly'  # for count
        else:
            self.yearly_ID_update = False  # for tracking
            self.read_csv_duration = 'all'  # for count

        # sub directories
        self.odirv = f'{self.odir}/V'
        self.odirt= f'{self.odir}/Vt'
        self.odirc = f'{self.odir}/Vtc'
        # read time interval and convert to second 
        # (re-convert to hour to output)
        self.dt = self.times.inferred_freq
        self.dtsec = conv_second(self.dt)
        self.dtH = self.dtsec / 3600.

        self.tlimit_dist = tlimit * self.dtH  # km
        self.tlimitR_dist = tlimit*7/12 * self.dtH  # km
        #self.normalized_distance = normalized_speed * self.dtH  # km

        # Execute program
        if not only_count:
            if self.ty == 'both' or self.ty == 'L':
                self._track('L')
            if self.ty == 'both' or self.ty == 'H':
                self._track('H')
        if self.ty == 'both' or self.ty == 'L':
            self._count('L')
        if self.ty == 'both' or self.ty == 'H':
            self._count('H')


    # --- start of tracking ----------
    def _track(self, ty):

        ttimes = self.times

        first = True
        last = False
        for i, tt0 in enumerate(ttimes):

            # next timestep
            try:
                tt1 = ttimes[i+1]
            except:
                last = True

            # check files exist
            if not self._check_file(tt0, ty):
                first = True
                continue

            if self._check_file(tt1, ty):
                exist_t1 = True
            else:
                exist_t1 = False

            print(f'{tt0}, {self.lev}, {ty}')

            if first:
                first = False

                if self.operational:  # read from existing
                    self.operational = False  # only first step
                    t0, ro0, So0, SBG0, SR0, lat0, lon0, ex0, l0 = self._extract_operational(tt0, ty)

                    ID_t0 = t0['ID']
                    ID_c = t0.at[0, '_ID_c']
                    SC_t0 = t0['_SC'].values
                    DC_t0 = t0['_DC'].values
                    SPLIT_by_ID_t0 = t0['SPLIT'].values
                    MERGE_to_ID = t0['MERGE'].values
                    LONs_B_t0 = t0['_LONs_B']
                    LATs_B_t0 = t0['_LATs_B']
                    prev_DIST = t0['DIST']

                else:  # init from none
                    t0, ro0, So0, SBG0, SR0, lat0, lon0, ex0, l0 = self._extract(tt0, ty)

                    ID_t0 = np.arange(1, l0+1, dtype='int32')
                    if l0 == 0:
                        first = True
                        continue

                    ID_c = ID_t0[-1]
                    SC_t0 = np.zeros((l0), dtype='int32')  # split ID carry
                    DC_t0 = np.full((l0), 1, dtype='int32')  # duration carry
                    SPLIT_by_ID_t0 = np.full((l0), -3, dtype='int32')
                    MERGE_to_ID = np.zeros((l0), dtype='int32')
                    LONs_B_t0 = np.full((l0), -3333, dtype='float32')
                    LATs_B_t0 = np.full((l0), -3, dtype='float32')
                    prev_DIST = np.full((l0), np.nan, dtype='float32')

            else:  # temporal succession (previous t1 -> current t0)
                t0 = t1
                ro0, So0, SBG0, SR0, lat0, lon0, ex0, l0 = ro1, So1, SBG1, SR1, lat1, lon1, ex1, l1
                ID_t0 = ID_t1
                SC_t0 = SC_t1  # split ID carry
                DC_t0 = DC_t1  # duration carry
                prev_DIST = DIST_t1
                SPLIT_by_ID_t0 = SPLIT_by_ID
                MERGE_to_ID = MERGE_to_ID_t1
                LONs_B_t0 = LONs_B
                LATs_B_t0 = LATs_B

                if last or not exist_t1:
                    t0['ID'] = ID_t0
                    t0['MERGE'] = np.full((l0), -3, dtype='int32')
                    t0['SPLIT'] = SPLIT_by_ID_t0
                    t0['DIST'] = np.full((l0), -2., dtype='float32')
                    t0['SPEED'] = np.full((l0), -2., dtype='float32')
                    t0['DIR'] = np.full((l0), -2., dtype='float32')

                    t0['_ID_c'] = ID_c
                    t0['_SC'] = SC_t0  # split ID carry
                    t0['_DC'] = DC_t0  # duration carry
                    t0['_LONs_B'] = LONs_B_t0
                    t0['_LATs_B'] = LATs_B_t0

                    t0 = t0.astype({'ID':'int32', 'MERGE':'int32', 'SPLIT':'int32'})

                    self._output_track(t0, tt0, ty)

                    if last:  break
                    else:  continue

            # count refresh (yearly v1.9.3)
            if tt1.month == 1 and tt1.day == 1 and tt1.hour == 0 and self.yearly_ID_update:
                ID_c = 0

            t1, ro1, So1, SBG1, SR1, lat1, lon1, ex1, l1 = self._extract(tt1, ty)

            ID_t1 = np.zeros((l1), dtype='int32')
            SC_t1 = np.zeros((l1), dtype='int32')  # split ID carry
            DC_t1 = np.zeros((l1), dtype='int32')  # duration carry

            MERGE_to_ID_t1 = np.zeros((l1), dtype='int32')
            SPLIT_by_ID = np.zeros((l1), dtype='int32')

            # integer array for primary assignment 
            PA_01 = np.full((l0), -1, dtype='int32')
            PA_10 = np.full((l1), -1, dtype='int32')
            # 2d arrays for temporally mutual characteristics
            D = np.full((l0,l1), 99999.9, dtype='float32')  # distance km
            P_01 = np.full((l0,l1), 99999.9, dtype='float32')  # 'penalty' in Lupo et al. (2023)
            P_10 = np.full((l0,l1), 99999.9, dtype='float32')  # 'penalty' in Lupo et al. (2023)

            # t0 loop for forward search
            for i in range(l0):

                # t1 loop to search Foward Primary Assignment (PA)
                #last_D = self.tlimit_dist
                #last_P = 99999.9
                _first = True
                for j in range(l1):

                    # calculate distance km
                    D[i,j] = gcd(lon0[i], lat0[i], lon1[j], lat1[j])

                    # check travel limit
                    if D[i,j] > self.tlimit_dist:  continue
                    
                    # check overlaping
                    if D[i,j] > ro0[i] + ro1[j]:  continue

                    # check smoothness of track (Lupo et al. 2023) (deprecated)
                    #if LONs_B_t0[i] >= 0:  # there is past track
                    #    # check directional change along lon and lat
                    #    lon_change = self._get_dlon(LONs_B_t0[i], lon0[i]) * self._get_dlon(lon0[i], lon1[j]) < 0
                    #    lat_change = (lat0[i] - LATs_B_t0[i]) * (lat1[j] - lat0[i]) < 0
                    #    if lon_change and lat_change:  # turning direction
                    #        if D[i,j] > self.tlimitR_dist:  continue

                    # calculate penalty (anti-similarity)
                    P_01[i,j] = self._calc_penalty(ro0[i], ro1[j],
                                                   So0[i], So1[j],
                                                   SBG0[i], SBG1[j],
                                                   #SR0[i], SR1[j],
                                                   D[i,j], prev_DIST[i])

                    if _first:
                        _first = False
                        PA_01[i] = j
                        last_P = P_01[i,j]
                        continue
                       
                    if P_01[i,j] <= last_P:  # most similar
                        PA_01[i] = j
                        last_P = P_01[i,j]

                if PA_01[i] == -1:  # there's no candidate: LYSIS
                    MERGE_to_ID[i] = -1  # solitary lysis
                    PA_01[i] = -2

            # t1 loop for backward search
            for j in range(l1):

                # t0 loop to search Backward Primary Assignment (PA)
                #last_D = self.tlimit_dist
                #last_P = 99999.9
                _first = True
                for i in range(l0):

                    # calculate distance (PF) km  # already calculated
                    #PF[i,j] = gcd(lon0[i], lat0[i], lon1[j], lat1[j])

                    # check travel limit
                    if D[i,j] > self.tlimit_dist:  continue

                    # check overlaping
                    if D[i,j] > ro0[i] + ro1[j]:  continue

                    # check smoothness (Lupo et al. 2023) (deprecated)
                    #if LONs_B_t0[i] >= 0:  # there is past track
                    #    # check directional change along lon and lat
                    #    lon_change = self._get_dlon(LONs_B_t0[i], lon0[i]) * self._get_dlon(lon0[i], lon1[j]) < 0
                    #    lat_change = (lat0[i] - LATs_B_t0[i]) * (lat1[j] - lat0[i]) < 0
                    #    if lon_change and lat_change:
                    #        # turning direction
                    #        if D[i,j] > self.tlimitR_dist:  continue

                    # calculate penalty (anti-similarity)
                    P_10[i,j] = self._calc_penalty(ro1[j], ro0[i],
                                                   So1[j], So0[i],
                                                   SBG1[j], SBG0[i],
                                                   #SR1[j], SR0[i],
                                                   D[i,j], np.nan)

                    if _first:
                        _first = False
                        PA_10[j] = i
                        last_P = P_10[i,j]
                    #if D[i,j] <= last_D:  # nearest (deprecated)
                    #    PA_10[j] = i
                    #    last_D = D[i,j]
                    if P_10[i,j] <= last_P:  # most similar
                        PA_10[j] = i
                        last_P = P_10[i,j]

                if PA_10[j] == -1:  # there's no candidate: solitary GENESIS
                    SPLIT_by_ID[j] = -1
                    PA_10[j] = -2

                    # genesis routine
                    ID_c = self._add_count(ID_c, ID_t0)
                    ID_t1[j] = ID_c
                    DC_t1[j] = 1  # duration carry (new life)

            # determine maintain
            for i in range(l0):
                if MERGE_to_ID[i] == -1:  continue  # solitary lysis

                for j in range(l1):
                    if SPLIT_by_ID[j] == -1:  continue  # solitary genesis

                    # check near
                    #if D[i,j] > self.tlimit_dist:  continue

                    if PA_01[i] == j and PA_10[j] == i:  # MAINTAIN

                        # maintain routine
                        ID_t1[j] = ID_t0[i]
                        SC_t1[j] = SC_t0[i]  # split ID carry
                        DC_t1[j] = DC_t0[i] + 1  # duration carry (keep tracking)

            # determine split (and rest of IDs for t1)
            for i in range(l0):
                if MERGE_to_ID[i] == -1:  continue  # solitaran lysis

                for j in range(l1):
                    if SPLIT_by_ID[j] == -1:  continue  # solitary genesis

                    # check near (deprdcated)
                    #if D[i,j] > self.tlimit_dist:  continue

                    if PA_01[i] != j and PA_10[j] == i:  # SPLIT
                        # genesis routine
                        ID_c = self._add_count(ID_c, ID_t0)
                        ID_t1[j] = ID_c
                        DC_t1[j] = 1  # duration carry (new life)

                        # split routine
                        SPLIT_by_ID[j] = ID_t0[i]  # split genesis
                        SC_t1[j] = ID_t0[i]  # split ID carry
                        SPLIT_by_ID_t0[i] = -2  # split origin

            # determine merge
            for i in range(l0):
                if MERGE_to_ID[i] == -1:  continue  # solitary lysis

                for j in range(l1):
                    if SPLIT_by_ID[j] == -1:  continue  # solitary genesis

                    # check near (deprecated)
                    #if D[i,j] > self.tlimit_dist:  continue

                    if ID_t1[j] == ID_t0[i]:
                        if i == 4:
                            _check_maintain = 1
                        continue  # maintain

                    if PA_01[i] == j and PA_10[j] != i:  # MERGE

                        MERGE_to_ID[i] = ID_t1[j]
                        MERGE_to_ID_t1[j] = -2  # merge origin

            # solve norikae (change-train) problem (1)
            for i in range(l0):
                if MERGE_to_ID[i] in [-1,-3,-4]:  continue

                for j in range(l1):
                    if SPLIT_by_ID[j] in [-1,-3,-4]:  continue

                    # check near (deprecated)
                    #if D[i,j] > self.tlimit_dist:  continue

                    # track steal by MERGE-SPLIT
                    if MERGE_to_ID[i] == ID_t1[j] and SPLIT_by_ID[j] > 0:
                        # overwrite maintain
                        PA_01[i] = j
                        PA_10[j] = i
                        _prev_ID_t1 = ID_t1[j]
                        ID_t1[j] = ID_t0[i]
                        SC_t1[j] = SC_t0[i]  # split ID carry
                        DC_t1[j] = DC_t0[i] + 1  # duration carry (keep tracking)
                        # overwrite split and merge to norikae flag
                        SPLIT_by_ID[j] = -4
                        MERGE_to_ID[i] = -4
                        # overwrite merge of others if any
                        if MERGE_to_ID_t1[j] == -2:
                            for ii in range(l0):
                                if MERGE_to_ID[ii] == _prev_ID_t1:
                                    MERGE_to_ID[ii] = ID_t1[j]

                    if j == 5:
                        print("norikae1.1",ID_t1[5])
                        print("check merge to id t1",MERGE_to_ID_t1[5])

                    # track steal by SPLIT-MERGE
                    if SPLIT_by_ID[j] == ID_t0[i] and MERGE_to_ID[i] > 0:
                        # overwrite maintain
                        PA_01[i] = j
                        PA_10[j] = i
                        _prev_ID_t1 = ID_t1[j]
                        ID_t1[j] = ID_t0[i]
                        SC_t1[j] = SC_t0[i]  # split ID carry
                        DC_t1[j] = DC_t0[i] + 1  # duration carry (keep tracking)
                        # overwrite split and merge to norikae flag
                        SPLIT_by_ID[j] = -4
                        MERGE_to_ID[i] = -4
                        # overwrite merge of others if any
                        if MERGE_to_ID_t1[j] == -2:
                            for ii in range(l0):
                                if MERGE_to_ID[ii] == _prev_ID_t1:
                                    MERGE_to_ID[ii] = ID_t1[j]

            # solve norikae (change-train) problem (2)
            #if self.norikae2:
            for j in range(l1):
                if SPLIT_by_ID[j] in [-1,-3,-4]:  continue

                for i in range(l0):
                    if MERGE_to_ID[i] in [-1,-3,-4]:  continue

                    # check near (deprecated)
                    #if D[i,j] > self.tlimit_dist:  continue

                    # check marge pair
                    if MERGE_to_ID[i] != ID_t1[j]:  continue
                    
                    # check merge to original
                    if ID_t0[i] != SC_t1[j]:  continue

                    # check duration to save longer
                    if DC_t0[i] < DC_t1[j]:  continue

                    # check extremely near (deprecated)
                    #if D[i,j] > ro0[i] or D[i,j] > ro1[j]:  continue

                    ID_t1[j] = ID_t0[i]  # overwrite ID to update MAINTAIN
                    MERGE_to_ID[PA_10[j]] = ID_t1[j]  # update previous MAINTAIN to MERGE
                    DC_t1[j] = DC_t0[i] + 1  # duration carry (keep tracking)
                    PA_01[i] = j  # update primary assign
                    PA_10[j] = i
                    # overwrite split and merge to norikae flag
                    SPLIT_by_ID[j] = -4
                    MERGE_to_ID[i] = -4

            if 0 in ID_t1:
                print("missed allocation of IDs for future detection features")
                exit()

            # moving speed and direction
            DIST = np.full((l0), np.nan, dtype='float32')
            DIST_t1 = np.full((l1), np.nan, dtype='float32')
            SPEED = np.full((l0), np.nan, dtype='float32')
            DIR = np.full((l0), np.nan, dtype='float32')

            # calc moving speed and direction
            for i in range(l0):

                is_B = LONs_B_t0[i] > -180.0
                is_F = MERGE_to_ID[i] in [0,-2]  # MAINTAIN or MERGEorigin

                if is_B and is_F:  # center difference # moving speed is nan at first and last 
                    lon_B = LONs_B_t0[i]
                    lat_B = LATs_B_t0[i]
                    lon_F = lon1[PA_01[i]]
                    lat_F = lat1[PA_01[i]]
                    fct = 0.5                
                    DIST[i] = gcd(lon_B, lat_B, lon_F, lat_F) * fct
                    DIST_t1[PA_01[i]] = DIST[i]
                    SPEED[i] = DIST[i] * 1000 / self.dtsec  # m/s
                    if SPEED[i] < 2.:  continue  # too slow to determine agle
                    DIR[i] = moving_direction(lon_B, lat_B, lon_F, lat_F)

            # store t0 lon/lat
            LONs_B = np.full((l1), -3333, dtype='float32')
            LATs_B = np.full((l1), -3, dtype='float32')
            for j in range(l1):
                # only maintin have values
                #if SPLIT_by_ID[j] == 0:
                if SPLIT_by_ID[j] in [0,-2]:
                    LONs_B[j] = lon0[PA_10[j]]
                    LATs_B[j] = lat0[PA_10[j]]
                else:
                    LONs_B[j] = -1111
                    LATs_B[j] = -1

            t0['ID'] = ID_t0
            t0['MERGE'] = MERGE_to_ID
            t0['SPLIT'] = SPLIT_by_ID_t0
            t0['DIST'] = DIST
            t0['SPEED'] = SPEED
            t0['DIR'] = DIR

            # meta for operational use
            t0['_ID_c'] = ID_c  # global ID
            t0['_SC'] = SC_t0  # split ID carry
            t0['_DC'] = DC_t0  # duration carry
            t0['_LONs_B'] = LONs_B_t0
            t0['_LATs_B'] = LATs_B_t0
            t0 = t0.astype({'ID':'int32', 'MERGE':'int32', 'SPLIT':'int32'})
            ids = t0['ID'].unique().tolist()
            self._output_track(t0, tt0, ty)

        if first:
            raise ValueError('no detection data, check detection (e.g., file path, thresholds, etc.)')

        #print('finish tracking!')


    def _calc_penalty(self,roA,roB,SoA,SoB,SBGA,SBGB,d,P_d):
        # Lupo et al. (2023, WAF)
        #N_ro = 632.0  # km
        #N_So = 13.6  # m/100km
        #N_SBG = 16.2  # m/100km
        #N_d = 800.0  # km
        #N_d = self.normalized_distance  # km
        #return np.abs(roA-roB)/roA + np.abs(SoA-SoB)/SoA + np.abs(SBGA-SBGB)/SBGA + d/N_d

        p1 = np.abs(roA-roB)/roA
        p2 = np.abs(SoA-SoB)/SoA
        p3 = np.abs(SBGA-SBGB)/SBGA
        #p4 = np.abs(SRA-SRB)/SRA
        p5 = d/P_d

        if self.penalty == 0:
            pp = p1+p2+p3
        elif self.penalty == 1:
            pp = p2

        # attempt to full stapshot normalization
        if P_d == 0.0:
            return d
        elif np.isnan(P_d):
            return pp 
        else:
            return pp+p5


    def _get_dlon(self, lon0, lon1):
        _dlon = lon1 - lon0
        # lon0 = 5,lon1 = -5 -> -10 ok
        # lon0 = 5, lon1 = 355 -> 350 -> -10
        # lon0 = -5, lon1 = 5 -> 10 ok
        # lon0 = 355, lon1 = 5 -> -350 -> 10
        # lon0 = -175, lon1 = 175 -> 350 -> -10
        if _dlon > 180.:
            return _dlon - 360.
        elif _dlon < -180.:
            return _dlon + 360.
        else:
            return _dlon


    def _add_count(self, ID_c, IDs):
        c = ID_c
        while True:
            c += 1
            if c in IDs: continue
            else: return c


    def _check_file(self, tt, ty):

        ts = f'{tt.year:04}{tt.month:02}{tt.day:02}{tt.hour:02}{tt.minute:02}'
        tsd = f'{tt.year:04}{tt.month:02}'

        if self.operational:
            f = f'{self.odirt}/{tsd}/V-{ty}-{ts}-{self.lev:04}.csv'
        else:
            f = f'{self.odirv}/{tsd}/V-{ty}-{ts}-{self.lev:04}.csv'

        return os.path.exists(f)


    def _extract_operational(self, tt, ty):

        ts = f'{tt.year:04}{tt.month:02}{tt.day:02}{tt.hour:02}{tt.minute:02}'
        tsd = f'{tt.year:04}{tt.month:02}'

        fn = f'{self.odirt}/{tsd}/V-{ty}-{ts}-{self.lev:04}'

        df = pd.read_csv(f'{fn}.csv', parse_dates=['time'])

        # use as ndarray to fasten
        ro = df['ro'].values.astype('f4')
        So = df['So'].values.astype('f4')
        SBG = df['SBG'].values.astype('f4')
        SR = df['SR'].values.astype('f4')
        lat = df['lat'].values.astype('f4')
        lon = df['lon'].values.astype('f4')
        ex = df['ex'].values.astype('i2')
        length = len(df)

        return df, ro, So, SBG, SR, lat, lon, ex, length


    def _extract(self, tt, ty):

        ts = f'{tt.year:04}{tt.month:02}{tt.day:02}{tt.hour:02}{tt.minute:02}'
        tsd = f'{tt.year:04}{tt.month:02}'

        fn = f'{self.odirv}/{tsd}/V-{ty}-{ts}-{self.lev:04}'

        df = pd.read_csv(f'{fn}.csv', parse_dates=['time'])

        # discard tracking parameter created by another analysis
        #df = df.filter(['time','ty','lev','lat','lon',
        #                'valV','valX','So','ro','Do',
        #                'SBG','SBGang','m','n','SR','ex',
        #                'EE','XX'])

        # use as ndarray to fasten
        ro = df['ro'].values.astype('f4')
        So = df['So'].values.astype('f4')
        SBG = df['SBG'].values.astype('f4')
        SR = df['SR'].values.astype('f4')
        lat = df['lat'].values.astype('f4')
        lon = df['lon'].values.astype('f4')
        ex = df['ex'].values.astype('i2')
        length = len(df)

        return df, ro, So, SBG, SR, lat, lon, ex, length


    def _output_track(self, df, tt, ty):

        ts = f'{tt.year:04}{tt.month:02}{tt.day:02}{tt.hour:02}{tt.minute:02}'
        tsd = f'{tt.year:04}{tt.month:02}'

        oo = f'{self.odirt}/{tsd}'
        os.makedirs(oo, exist_ok=True)
        df.to_csv(f'{oo}/V-{ty}-{ts}-{self.lev:04}.csv',
                  header=True, index=False, float_format='%.4f')

    # --- start of counting ----------
    def _count(self, ty):

        read_csv_duration = self.read_csv_duration
        ttimes = self.times
        dt = self.dt
        addmonth = self.addmonth

        if read_csv_duration == 'all':

            df = self._open_df(ttimes, ty)
            _ = self._count_main(df, ttimes, ty)

        elif read_csv_duration == 'monthly':  # monthly loop

            t_init = ttimes[0]
            t_end = ttimes[-1]
            monthly_counting_times = pd.date_range(t_init, t_end, freq='MS')
            last_hour = int(24 - self.dtH)

            first = True
            for t in monthly_counting_times:

                last_day = calendar.monthrange(t.year, t.month)[1]
                target_times = pd.date_range(f'{t.year}-{t.month:02}-01 00',
                                             f'{t.year}-{t.month:02}-{last_day:02} {last_hour:02}',
                                             freq=dt)

                nt = t + relativedelta(months=addmonth)

                if nt <= ttimes[-1]:

                    last_day = calendar.monthrange(nt.year, nt.month)[1]

                    # prepare df
                    if first:  # init df
                        first = False
                        first_months = pd.date_range(f'{t.year}-{t.month:02}-01 00',
                                                     f'{nt.year}-{nt.month:02}-{last_day:02} {last_hour:02}',
                                                     freq=dt)
                        df = self._open_df(first_months, ty)

                    elif nt.month == ttimes[-1].month: # last month to add
                        true_last_day = ttimes[-1].day
                        true_last_hour = ttimes[-1].hour
                        new_month = pd.date_range(f'{nt.year}-{nt.month:02}-01 00',
                                                  f'{nt.year}-{nt.month:02}-{true_last_day:02} {true_last_hour:02}',
                                                  freq=dt)
                        df_new = self._open_df(new_month, ty)
                        df = pd.concat([df_old, df_new], ignore_index=True)

                    else:  # continue with a constant time width (addmonth)
                        new_month = pd.date_range(f'{nt.year}-{nt.month:02}-01 00',
                                                  f'{nt.year}-{nt.month:02}-{last_day:02} {last_hour:02}',
                                                  freq=dt)
                        df_new = self._open_df(new_month, ty)
                        df = pd.concat([df_old, df_new], ignore_index=True)

                    # core process
                    df_old = self._count_main(df, target_times, ty)

                elif first:
                    raise ValueError('error: addmonth is exceeding a whole month to analyze. set long_term=False or change addmonth')

                else:
                    # residual process
                    df_old = self._count_main(df_old, target_times, ty)

        #print('finish counting!')
        

    def _count_main(self, _df, target_times, ty):

        read_csv_duration = self.read_csv_duration
        ttimes = self.times
        lev = self.lev

        df = _df.reset_index(drop=True)

        if read_csv_duration == 'all':
            ids = df['ID'].unique().tolist()
            #print(f'\r counting N={len(ids)} {lev} {ty}', end='')
            print(f'counting N={len(ids)} {lev} {ty}')

        elif read_csv_duration == 'monthly':
            Y = target_times[0].year
            M = target_times[0].month
            di = pd.DatetimeIndex(df['time'])
            df_1mo = df[(di.year == Y)&(di.month == M)]
            ids = df_1mo['ID'].unique().tolist()
            #print(f'\r counting {Y}-{M:02} N={len(ids)} {lev} {ty}', end='')
            print(f'counting {Y}-{M:02} N={len(ids)} {lev} {ty}')

            if Y == ttimes[0].year and M == ttimes[0].month:
                init_month = True
            else:
                init_month = False

        for ID in ids:
            cont_idx = df[df['ID'] == ID].index.tolist()
            df_ID = df.iloc[cont_idx].reset_index(drop=True)

            if df_ID.DU.values[0] < 99998.:
                continue  # already calc in the previous month

            _duration = len(df_ID) * self.dtH
            df.loc[cont_idx, 'DU'] = _duration  # update DU first

            if _duration < self.DU_thres:
                continue

            Sos = df_ID.So.values
            exs = df_ID.ex.values
            times = pd.to_datetime(df_ID.time.values)

            # init for max intensity section
            pre_max_So = Sos[0]
            l_max = 0
            #pre_ex = exs[0]
            l_gen, l_lys = [], []
            for l in range(len(df_ID)):

                _So = Sos[l]

                # check maximum development
                if _So > pre_max_So:
                    pre_max_So = _So
                    l_max = l

                if l == 0 or l == len(df_ID)-1:  # new system 20241021
                    continue
                else:
                    pre_ex = exs[l-1]
                    _ex = exs[l]
                    pos_ex = exs[l+1]

                    if pre_ex == 0 and _ex == 1:
                        l_gen.append(l)
                    if _ex == 1 and pos_ex == 0:
                        l_lys.append(l)

                ## check genesis/lysis of vortex from wave  # old system
                #if pre_ex == 0 and _ex == 1:
                #    l_gen.append(l)
                #if pre_ex == 1 and _ex == 0:
                #    l_lys.append(l)
                ## update pre_ex for next loop
                #pre_ex = _ex

            # collect track data
            max_idx = cont_idx[l_max]
            df.loc[max_idx, 'MAX'] = 1
            df.loc[cont_idx, 'MAXSo'] = pre_max_So
            df.loc[cont_idx, 'ALLDIST'] = df_ID['DIST'].sum()
            df.loc[cont_idx, 'MEDDIST'] = df_ID['DIST'].median()
            exGEN_idx = [cont_idx[i] for i in l_gen]
            exLYS_idx = [cont_idx[i] for i in l_lys]
            df.loc[exGEN_idx, 'exGEN'] = 1
            df.loc[exLYS_idx, 'exLYS'] = 1

            XS, qs_idx, qs_c = self._check_sequence(df_ID, cont_idx, 'XS')
            if qs_idx != []:
                if qs_c == 1:
                    df.loc[qs_idx, 'XS'] = XS
                    df.loc[cont_idx, 'MAXXS'] = XS
                if qs_c >= 2:
                    for c in range(qs_c):
                        df.loc[qs_idx[c], 'XS'] = XS[c]
                    df.loc[cont_idx, 'MAXXS'] = max(XS)

            QS, qs_idx, qs_c = self._check_sequence(df_ID, cont_idx, 0.95)
            if qs_idx != []:
                if qs_c == 1:
                    df.loc[qs_idx, 'QS5'] = QS
                    df.loc[cont_idx, 'MAXQS5'] = QS
                if qs_c >= 2:
                    for c in range(qs_c):
                        df.loc[qs_idx[c], 'QS5'] = QS[c]
                    df.loc[cont_idx, 'MAXQS5'] = max(QS)

            QS, qs_idx, qs_c = self._check_sequence(df_ID, cont_idx, 0.7)
            if qs_idx != []:
                if qs_c == 1:
                    df.loc[qs_idx, 'QS7'] = QS
                    df.loc[cont_idx, 'MAXQS7'] = QS
                if qs_c >= 2:
                    for c in range(qs_c):
                        df.loc[qs_idx[c], 'QS7'] = QS[c]
                    df.loc[cont_idx, 'MAXQS7'] = max(QS)

        #df = add_split_dus_qs(df, target_times)

        # remove noise
        df1 = df[df.DU > self.DU_thres].reset_index(drop=True)  # hour
        df2 = df1[df1.MAXSo > self.MAXSo_thres].reset_index(drop=True)
        df3 = df2[df2.ALLDIST > self.ALLDIST_thres].reset_index(drop=True)  # DIST / day

        #save data
        odir = self.odirc
        for t in target_times:

            _Y = t.year
            _M = t.month
            _D = t.day
            _H = t.hour
            _T = t.minute
            ym2 = f'{_Y}{_M:02}'
            tn = f'{ym2}{_D:02}{_H:02}{_T:02}'

            if not os.path.exists(f'{odir}/{ym2}'):
                os.makedirs(f'{odir}/{ym2}', exist_ok=True)

            di = pd.DatetimeIndex(df3['time'])
            df_o = df3[(di.year == _Y)&(di.month == _M)&(di.day == _D)&(di.hour == _H)&(di.minute == _T)]

            # remove meta
            #df_o = df_o.drop(columns=['_ID_c', '_SC', '_DC', '_LONs_B', '_LATs_B'])
            df_o = df_o.drop(columns=['_ID_c', '_SC', '_LONs_B', '_LATs_B'])

            df_o.to_csv(f'{odir}/{ym2}/V-{ty}-{tn}-{lev:04}.csv',
                        index=False, float_format='%.4f')

        if read_csv_duration == 'all':
            return df3
        elif read_csv_duration == 'monthly':
            if init_month:
                return df3
            else:
                prev_month = target_times[0] - relativedelta(months=1)
                _pM = prev_month.month
                di = pd.DatetimeIndex(df3['time'])
                return df3[di.month != _pM]


    def _open_df(self, tmp_time, ty):

        idir = self.odirt
        lev = self.lev
        first = True

        for t in tmp_time:

            ym2 = f'{t.year}{t.month:02}'
            tn = f'{ym2}{t.day:02}{t.hour:02}{t.minute:02}'

            if first:
                if os.path.isfile(f'{idir}/{ym2}/V-{ty}-{tn}-{lev:04}.csv'):
                    df = pd.read_csv(f'{idir}/{ym2}/V-{ty}-{tn}-{lev:04}.csv', parse_dates=['time'])
                    first = False
                else:
                    print(f'missing {idir}/{ym2}/V-{ty}-{tn}-{lev:04}.csv')
                    continue
            else:
                _df = pd.read_csv(f'{idir}/{ym2}/V-{ty}-{tn}-{lev:04}.csv', parse_dates=['time'])
                df = pd.concat([df, _df], ignore_index=True)

        if first:
            raise ValueError('there is no object, stop, ', lev, ty)

        lendf = len(df)

        df['DU'] = np.full((lendf), 99999.9, dtype='f4')
        df['XS'] = np.zeros((lendf), dtype='f4')
        df['QS5'] = np.zeros((lendf), dtype='f4')
        df['QS7'] = np.zeros((lendf), dtype='f4')

        df['ALLDIST'] = np.full((lendf), 99999.9, dtype='f4')
        df['MEDDIST'] = np.zeros((lendf), dtype='f4')
        df['MAX'] = np.zeros((lendf), dtype='i4')
        df['MAXSo'] = np.full((lendf), 99999.9, dtype='f4')
        df['MAXXS'] = np.zeros((lendf), dtype='f4')
        df['MAXQS5'] = np.zeros((lendf), dtype='f4')
        df['MAXQS7'] = np.zeros((lendf), dtype='f4')
        df['exGEN'] = np.zeros((lendf), dtype='i4')
        df['exLYS'] = np.zeros((lendf), dtype='i4')

        return df


    def _checkqs3(self, lon1, lat1, r1, lon2, lat2, r2, O):

        d = gcd(lon1, lat1, lon2, lat2)

        if r1 < r2:
            c1 = circle(r=r1, x=0)
            c2 = circle(r=r2, x=d)
        else:
            c1 = circle(r=r2, x=0)
            c2 = circle(r=r1, x=d)

        o = circle_overlap(c1, c2)
        overlap_ratio = o.overlap

        return overlap_ratio > O

    def _check_sequence(self, _df, idx, O):

        qst_du_tmp = 0
        qs_idx_tmp = []
        seq0 = True
        seq = False
        qs_c = 0

        prev_cond = False

        # get data
        lons = _df.lon.values
        lats = _df.lat.values
        ros = _df.ro.values.astype('f4')
        exs = _df.ex.values

        for i in range(len(_df)):

            if O == 'XS':
                _cond = exs[i] == 1

            elif i != len(_df) - 1:  # QS
                lon0, lon1 = lons[i], lons[i+1]
                lat0, lat1 = lats[i], lats[i+1]
                ro0, ro1 = ros[i], ros[i+1]

                now_cond = self._checkqs3(lon0, lat0, ro0, lon1, lat1, ro1, O)
                _cond = prev_cond or now_cond
                prev_cond = _cond
            else:
                _cond = False

            if _cond:
                qst_du_tmp += self.dtH
                qs_idx_tmp.append(idx[i])
                seq = True

                if seq0:
                    qs_c += 1
                    seq0 = False

            end_of_seq = not _cond and seq
            is_last = i == len(_df) - 1 and seq

            if end_of_seq or is_last:

                if qs_c == 1:
                    qst_du = qst_du_tmp
                    qs_idx = qs_idx_tmp

                if qs_c == 2:
                    qst_du = [qst_du, qst_du_tmp]
                    qs_idx = [qs_idx, qs_idx_tmp]

                if qs_c >= 3:
                    qst_du.append(qst_du_tmp)
                    qs_idx.append(qs_idx_tmp)

                qst_du_tmp = 0
                qs_idx_tmp = []

                seq0 = True
                seq = False

        if qs_c == 0:
            qst_du = 0
            qs_idx = []

        return qst_du, qs_idx, qs_c
