#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Mar 18 20:20:54 2022

@author: aguimera
"""

import datetime
from PyGFETdb.ws_json_class import ws_json
from PyGFETdb.DataClass import DataCharAC
import pickle
import sys
import pandas as pd
import numpy as np
import traceback
import base64
from cryptography.fernet import Fernet
import importlib
from multiprocessing import Pool


def WS_Types(name):
    if name.endswith('id'):
        return 'int'
    elif name == 'Data':
        return 'bin'
    else:
        return 'str'


class PyFETdb():
    Threads = 8

    def __init__(self, connection, Multiprocess=True):
        f = Fernet(connection)
        # da = f.decrypt(open('./Connection', 'rb').read())
        da = f.decrypt(importlib.resources.read_binary('PyGFETdb', 'Connection.bin'))
        self.connection = pickle.loads(da)
        self._DEBUG = False
        self.Multiprocess = Multiprocess

    def CreateQueryConditions(self, Conditions):
        Cond = []
        values = []
        for c in Conditions:
            cc = " %s OR ".join([c] * len(Conditions[c]))
            Cond.append("({} %s)".format(cc))
            for v in Conditions[c]:
                values.append(v)
        sCond = ' AND '.join(Cond)
        return sCond, tuple(values)

    def ParseQuery(self, query, values):
        sInds = []
        sInd = query.find('%s')
        while sInd > 0:
            sInds.append(sInd)
            sInd = query.find('%s', sInd + 1)

        if len(sInds) > len(values):
            print("Error parsing query")
            traceback.print_exc()

        pVals = []
        for v in values:
            if type(v) == str:
                if v.startswith('str_to_date'):
                    pVals.append(v)
                else:
                    pVals.append("'{}'".format(v))
            elif type(v) == bytes:
                pVals.append("'{}'".format(base64.b64encode(v).decode()))
            else:
                pVals.append(str(v))

        return query % tuple(pVals)

    def _FormatOutputQuery(self, Output):
        Output2 = []
        OutF = []
        for o in Output:
            OutF.append(o.replace('.', '_'))
            Output2.append('{} AS {}'.format(o, o.replace('.', '_')))
        return OutF, ' , '.join(Output2)

    def _decode(self, Res):
        for r in Res:
            for k, v in r.items():
                if k.endswith('Data'):
                    continue
                r[k] = v.decode()
        return Res

    def _execute(self, query, values=None, oper='query'):
        if oper == 'query':
            q = self.ParseQuery(query, values)
        else:
            q = query

        if self._DEBUG:
            print('Debug Query, Operation: ', oper)
            print(q)

        Res = ws_json.call(self.connection, q, oper)
        if Res['error']:
            # print(q)
            print('Error in call ', Res['errorCode'])
            return None

        return self._decode(Res['result'])

    def MultiSelect(self, Table, Conditions, Output, Order=None):
        OutF, Out = self._FormatOutputQuery(Output)
        Cond, Values = self.CreateQueryConditions(Conditions)
        query = "SELECT {} FROM {} WHERE {}".format(Out, Table, Cond)
        if Order:
            query = '{} ORDER BY {}'.format(query, Order)

        return self._execute(query, Values)

    def GetId(self, Table, Value, Field='Name', NewVals=None):
        query = "SELECT * FROM {} WHERE {} = %s".format(Table, Field)
        Res = self._execute(query, (Value,))

        if len(Res) == 0:
            if NewVals:
                ID = self.NewRow(Table=Table, Fields=NewVals)
            else:
                ID = None
        elif len(Res) == 1:
            ID = Res[0]['id{}'.format(Table)]
        elif len(Res) > 1:
            print('Warning', query, Res)
            ID = Res[0]['id{}'.format(Table)]

        return int(ID)

    def NewRow(self, Table, Fields):
        data = {}
        data['table'] = Table
        data['fields'] = {}
        for ic, (k, v) in enumerate(Fields.items()):
            data['fields'][ic] = {'name': k,
                                  'value': v,
                                  'type': WS_Types(k)}
        Res = self._execute(data, oper='insert')
        return int(Res[0]['id'])

    def UpdateRow(self, Table, Fields, Condition):
        data = {}
        data['table'] = Table
        data['fields'] = {}
        data['where'] = Condition
        for ic, (k, v) in enumerate(Fields.items()):
            data['fields'][ic] = {'name': k,
                                  'value': v,
                                  'type': WS_Types(k)}
        Res = self._execute(data, oper='update')
        return int(Res[0]['id'])

    def GetTrtsInfo(self, Conditions, Output=None):
        if Output is None:
            Output = ('Trts.idTrts',
                      'Trts.Name',
                      'TrtTypes.idTrtTypes',
                      'TrtTypes.Name',
                      'TrtTypes.Contact',
                      'TrtTypes.Width',
                      'TrtTypes.Length')

        OutF, Out = self._FormatOutputQuery(Output)
        Cond, Values = self.CreateQueryConditions(Conditions)
        query = """ SELECT distinct {}
                    FROM VTrts, Trts
                    INNER JOIN Devices ON Devices.idDevices = Trts.Device_id
                    INNER JOIN Wafers ON Wafers.idWafers = Devices.Wafer_id
                    INNER JOIN TrtTypes ON TrtTypes.idTrtTypes = Trts.Type_id
                    WHERE VTrts.idTrts = Trts.idTrts and  {} """.format(Out,
                                                                        Cond)

        return self._execute(query, Values)

    def GetCharactInfo(self, Table, Conditions, Output):
        OutF, Out = self._FormatOutputQuery(Output)
        Cond, Values = self.CreateQueryConditions(Conditions)

        query = """ SELECT distinct {Out}
                    FROM {Table}
                    INNER JOIN Trts ON Trts.idTrts = {Table}.Trt_id
                    INNER JOIN TrtTypes ON TrtTypes.idTrtTypes = Trts.Type_id
                    INNER JOIN Devices ON Devices.idDevices = Trts.Device_id
                    INNER JOIN Wafers ON Wafers.idWafers = Devices.Wafer_id
                    INNER JOIN VTrts ON VTrts.idTrts = Trts.idTrts
                    INNER JOIN Users ON {Table}.User_id = Users.idUsers
                    WHERE {Cond}
                    ORDER BY {Table}.MeasDate; """.format(Out=Out,
                                                          Table=Table,
                                                          Cond=Cond
                                                          )

        return self._execute(query, Values)

    def GetCharactFromId(self, Table, Id, GetGate=False):
        DataF = '{}.Data'.format(Table)
        Output = ['Trts.Name',
                  'Devices.Name',
                  'Devices.Comments',
                  'Wafers.Name',
                  'Wafers.Run',
                  'Wafers.Substrate',
                  'Wafers.Masks',
                  'Wafers.Graphene',
                  'Wafers.Comments',
                  'TrtTypes.Name',
                  'TrtTypes.Width',
                  'TrtTypes.Length',
                  'TrtTypes.Pass',
                  'TrtTypes.Area',
                  'TrtTypes.Contact',
                  'TrtTypes.Shape',
                  'Users.Name',
                  DataF,
                  '{}.Ph'.format(Table),
                  '{}.Solution'.format(Table),
                  '{}.IonStrength'.format(Table),
                  '{}.IsCmp'.format(Table),
                  '{}.FuncStep'.format(Table),
                  '{}.AnalyteCon'.format(Table),
                  '{}.Comments'.format(Table)]

        if GetGate:
            if Table == 'DCcharacts':
                GidF = 'DCcharacts.Gate_id'
                Output.append(GidF)
            else:
                GidF = None
                DCidF = 'ACcharacts.DC_id'
                Output.append(DCidF)

        OutF, Out = self._FormatOutputQuery(Output)

        query = """ SELECT {Out}
                    FROM {Table}
                    INNER JOIN Trts ON Trts.idTrts = {Table}.Trt_id
                    INNER JOIN TrtTypes ON TrtTypes.idTrtTypes = Trts.Type_id
                    INNER JOIN Devices ON Devices.idDevices = Trts.Device_id
                    INNER JOIN Wafers ON Wafers.idWafers = Devices.Wafer_id
                    INNER JOIN VTrts ON VTrts.idTrts = Trts.idTrts
                    INNER JOIN Users ON {Table}.User_id = Users.idUsers
                    WHERE  {Table}.id{Table} = %s """.format(Out=Out,
                                                             Table=Table,
                                                             )
        res = self._execute(query, (Id,))
        if len(res) == 0:
            print('Id Not found')
            return None
        if len(res) > 1:
            print('Warning duplicated ID !!!!')

        Res = res[0]

        Dat = {}
        for f in OutF:
            if f.endswith('Data'):
                Dat.update(pickle.loads(Res[f], encoding='latin'))
                continue
            fp = f.split('_')
            if fp[0] == Table:
                fn = Dat.get('Info', {})
                fn[fp[1]] = Res[f]
                Dat['Info'] = fn
            elif fp[0] == 'Users':
                fn = Dat.get('Info', {})
                fn['User' + fp[1]] = Res[f]
                Dat['Info'] = fn
            else:
                fn = Dat.get(fp[0], {})
                fn[fp[1]] = Res[f]
                Dat[fp[0]] = fn

        Dat['Name'] = Res['Trts_Name']

        # Add DC data to ACcharact
        if Table == 'ACcharacts':
            cond = {'idDCcharacts = ': (Res['ACcharacts_DC_id'],)}
            ResDC = self.MultiSelect(Table='DCcharacts',
                                     Conditions=cond,
                                     Output=('Data',))
            DCData = pickle.loads(ResDC[0]['Data'], encoding='latin')
            Dat['VgsDC'] = DCData.pop('Vgs')
            Dat['VdsDC'] = DCData.pop('Vds')
            if 'Ids' not in Dat:
                # print('fill DC ids')
                Dat['Ids'] = DCData.pop('Ids')
            Dat['DCData'] = DCData

        if GetGate:
            if Table == 'DCcharacts':
                Dat['Gate'] = self.GetGateFromId(Res['DCcharacts_Gate_id'])
            elif Table == 'ACcharacts':
                cond = {'idDCcharacts = ': (Res['ACcharacts_DC_id'],)}
                Res = self.MultiSelect(Table='DCcharacts',
                                       Conditions=cond,
                                       Output=('Gate_id',))

                Dat['Gate'] = self.GetGateFromId(Res[0]['Gate_id'])

        return Dat

    def GetGateFromId(self, idg):
        Rows = self.MultiSelect(Table='Gcharacts',
                                Conditions={'idGcharacts=': (idg,)},
                                Output=('Data',))
        if len(Rows):
            GateData = pickle.loads(Rows[0]['Data'], encoding='latin')
            Res = self.GetCharactInfo(Table='DCcharacts',
                                      Conditions={'DCcharacts.Gate_id = ': (idg,)},
                                      Output=('Trts.Name',))
            Trts = []
            for re in Res:
                Trts.append(re['Trts_Name'])

            GateData['GateTrts'] = Trts
            return GateData
        else:
            return None

    def GetData2(self, Conditions, Table, Last=True, GetGate=False):
        Output = ('id{}'.format(Table, Table, ),
                  'MeasDate'.format(Table),
                  'Trts.Name')

        Res = self.GetCharactInfo(Table=Table,
                                  Conditions=Conditions,
                                  Output=Output)

        if len(Res) == 0:
            return []

        pdSers = []
        for r in Res:
            pdSers.append(pd.Series(r))

        DTypes = {'id{}'.format(Table): int,
                  'MeasDate': 'datetime64[ns]',
                  'Trts_Name': str}
        dfr = pd.concat(pdSers, axis=1).transpose().astype(DTypes)

        if Last:
            gn = dfr.groupby('Trts_Name')
            ids = []
            for TrtName, dt in gn:
                dt.sort_values(by='MeasDate', ascending=False, inplace=True)
                ids.append(dt.iloc[0]['id{}'.format(Table)])
        else:
            ids = list(dfr['id{}'.format(Table)])

        if self.Multiprocess:
            Args = []
            for Id in ids:
                Args.append((Table, Id, GetGate))
            print("Downloading {} Records".format(len(ids)))
            with Pool(self.Threads) as p:
                Data = p.starmap(self.GetCharactFromId, Args)
        else:
            Data = []
            for ic, Id in enumerate(ids):
                print("Downloading {} of {}".format(ic, len(ids)))
                Data.append(self.GetCharactFromId(Table=Table,
                                                  Id=Id,
                                                  GetGate=GetGate))

        return Data

    def InsertCharact(self, DCVals, Fields, ACVals=None, OptFields=None,
                      TrtTypeFields=None, OverWrite=True):

        Author = Fields['User']
        Wafer = Fields['Wafer']
        Device = Fields['Device']
        TrtType = Fields['TrtType']
        Trt = Fields['Trt']

        if not Wafer.replace('-', '').isalnum():
            print('ABORTED')
            print('Wafer name not valid characters')
            return
        if not Device.replace('-', '').isalnum():
            print('ABORTED')
            print('Device name not valid characters')
            return
        if not Trt.replace('-', '').isalnum():
            print('ABORTED')
            print('Trt name not valid characters')
            return
        if len(Wafer) > 12:
            print('ABORTED')
            print('Wafer name too long, 12 allowed')
            return
        if len(Device) > 20:
            print('ABORTED')
            print('Device name too long, 20 allowed')
            return
        if len(Trt) > 25:
            print('ABORTED')
            print('Trt name too long, 25 allowed')
            return

        # Set Get information in other tables
        Author_id = self.GetId(Table='Users',
                               Value=Author,
                               NewVals={'Name': Author})

        TrtType_id = self.GetId(Table='TrtTypes',
                                Value=TrtType,
                                NewVals=TrtTypeFields)

        Wafer_id = self.GetId(Table='Wafers',
                              Value=Wafer,
                              NewVals={'Name': Wafer})

        Device_id = self.GetId(Table='Devices',
                               Value=Device,
                               NewVals={'Name': Device,
                                        'Wafer_id': Wafer_id})

        Trt_id = self.GetId(Table='Trts',
                            Value=Trt,
                            NewVals={'Name': Trt,
                                     'Device_id': Device_id,
                                     'Type_id': TrtType_id})

        TimeNow = datetime.datetime.now()

        #######################################################################
        # Insert DC data
        #######################################################################
        # FILL NewData structure
        NewData = {'Trt_id': Trt_id,
                   'User_id': Author_id,
                   'Data': pickle.dumps(DCVals),
                   'MeasDate': str(DCVals['DateTime']),
                   'UpdateDate': str(TimeNow),
                   'Gate_id': Fields['Gate_id']}

        if 'IsOK' in DCVals:
            NewData['IsOK'] = int(DCVals['IsOK'])  # DCVals['IsOK']
        if OptFields:
            NewData.update(OptFields)

        # Check if exists
        sMeasDate = DCVals['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
        Rows = self.MultiSelect(Table='DCcharacts',
                                Conditions={'Trt_id=': (Trt_id,),
                                            'MeasDate=': (sMeasDate,)},
                                Output=('Trt_id', 'idDCcharacts'))

        if len(Rows) == 0:  # New Record
            DCchatact_id = self.NewRow(Table='DCcharacts', Fields=NewData)
        else:
            print('WARNING EXISTS', Rows)
            if OverWrite:  # OverWrite
                DCchatact_id = Rows[0]['idDCcharacts']
                print('Overwrite Record id ', DCchatact_id)
                scond = 'idDCcharacts = {}'.format(DCchatact_id)
                self.UpdateRow(Table='DCcharacts',
                               Fields=NewData,
                               Condition=scond)

        NewData.pop('Gate_id')
        #######################################################################
        # Insert DC data
        #######################################################################
        if ACVals:
            # FILL NewData structure
            NewData = {'Trt_id': Trt_id,
                       'User_id': Author_id,
                       'UpdateDate': str(TimeNow),
                       'DC_id': DCchatact_id,
                       'Data': pickle.dumps(ACVals),
                       'MeasDate': str(ACVals['DateTime'])}
            if 'IsOK' in ACVals:
                NewData['IsOK'] = int(ACVals['IsOK'])  # ACVals['IsOK']
            if OptFields:
                NewData.update(OptFields)

            # Check if exists
            sMeasDate = ACVals['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
            Rows = self.MultiSelect(Table='ACcharacts',
                                    Conditions={'Trt_id=': (Trt_id,),
                                                'MeasDate=': (sMeasDate,)},
                                    Output=('Trt_id', 'idACcharacts'))
            if len(Rows) == 0:  # New Record
                self.NewRow(Table='ACcharacts', Fields=NewData)
            else:
                print('WARNING EXISTS', Rows)
                if OverWrite:  # OverWrite
                    ACchatact_id = Rows[0]['idACcharacts']
                    print('Overwrite Record id ', ACchatact_id)
                    scond = 'idACcharacts = {}'.format(ACchatact_id)
                    self.UpdateRow(Table='ACcharacts',
                                   Fields=NewData,
                                   Condition=scond)

    def InsertGateCharact(self, GateData, Fields, OverWrite=True):
        # Set Get information in other tables
        Author = Fields['User']
        Author_id = self.GetId(Table='Users',
                               Value=Author,
                               NewVals={'Name': Author})
        Fields.pop('User')

        # Check if exists
        sMeasDate = GateData['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
        Rows = self.MultiSelect(Table='Gcharacts',
                                Conditions={'Name=': (Fields['Name'],),
                                            'MeasDate=': (sMeasDate,)},
                                Output=('idGcharacts',))

        NewData = {'Data': pickle.dumps(GateData),
                   'User_id': Author_id,
                   'MeasDate': str(GateData['DateTime']),
                   'UpdateDate': str(datetime.datetime.now())}
        NewData.update(Fields)

        if len(Rows) == 0:  # New Record
            Gate_id = self.NewRow(Table='Gcharacts', Fields=NewData)
        else:
            Gate_id = Rows[0]['idGcharacts']
            print('Gate id foung {}, Overwriting'.format(Gate_id))
            if OverWrite:  # OverWrite                
                # print ('WARNING EXISTS', Rows)
                # print ('Overwrite Record id ', Gate_id)
                scond = 'idGcharacts = {}'.format(Gate_id)
                self.UpdateRow(Table='Gcharacts',
                               Fields=NewData,
                               Condition=scond)
        return Gate_id

    def InsertDataFrame(self, dfUpload):
        FieldsCols = ('User',
                      'Wafer',
                      'Device',
                      'TrtType',
                      'Trt')

        DataFields = ('DCVals', 'ACVals', 'GateData', 'TrtTypeFields')
        NonOptFields = list(FieldsCols) + list(DataFields)

        for ir, r in dfUpload.iterrows():
            print("Upload {} of {}  {}".format(ir, dfUpload.shape[0], r.Trt))

            GateId = None
            if r.GateData is not None:
                GateFields = {'User': r.User,
                              'Name': '{}-Gate'.format(r.Device),
                              'FileName': r.FileName}
                GateId = self.InsertGateCharact(GateData=r.GateData,
                                                Fields=GateFields)

            Fields = r[list(FieldsCols)].to_dict()
            Fields['Gate_id'] = GateId

            self.InsertCharact(DCVals=r.DCVals,
                               Fields=Fields,
                               ACVals=r.ACVals,
                               OptFields=r.drop(NonOptFields).to_dict(),
                               TrtTypeFields=r.TrtTypeFields)
    # Inside class PyFETdb:
    def _init_caches(self):
        self._cache = {
            'Users': {},
            'TrtTypes': {},
            'Wafers': {},
            'Devices': {},
            'Trts': {},
            'Gates': {},  # key -> idGcharacts
            'dc_exists': set(),  # (Trt_id, sMeasDate)
            'ac_exists': set(),  # (Trt_id, sMeasDate)
        }

    def _getid_cached(self, table, value, field='Name', newvals=None):
        cache = self._cache[table]
        if value in cache:
            return cache[value]
        _id = self.GetId(Table=table, Value=value, Field=field, NewVals=newvals)
        cache[value] = _id
        return _id

    def InsertDataFrame_fast(self, dfUpload, chunk_size_exists=200):
        """
        Faster version of InsertDataFrame for ~1k rows:
        - Caches FK ids (Users, TrtTypes, Wafers, Devices, Trts)
        - Caches Gate ids
        - Pre-checks existence of DC/AC in batches
        - Uses itertuples() and minimizes per-row prints
        """
        # 1) init caches
        if not hasattr(self, "_cache"):
            self._init_caches()
        cache = self._cache
        # Light local refs
        _getid_cached = self._getid_cached
        NonOptFields = ('User', 'Wafer', 'Device', 'TrtType', 'Trt', 'DCVals', 'ACVals', 'GateData', 'TrtTypeFields',
                        'FileName')

        # 2) Collect unique FK values
        users_u = set(dfUpload['User'].dropna().astype(str))
        wafers_u = set(dfUpload['Wafer'].dropna().astype(str))
        devices_u = set(dfUpload['Device'].dropna().astype(str))
        trttype_u = set(dfUpload['TrtType'].dropna().astype(str))
        trt_u = set(dfUpload['Trt'].dropna().astype(str))

        # 3) Resolve FK ids once per unique value (still uses your GetId, but cached)
        for u in users_u:
            _getid_cached('Users', u, newvals={'Name': u})
        for w in wafers_u:
            _getid_cached('Wafers', w, newvals={'Name': w})
        for tt in trttype_u:
            # We don't know full TrtTypeFields here; we'll still cache per name.
            # If some rows bring richer fields, the first insert will set them.
            _getid_cached('TrtTypes', tt, newvals={'Name': tt})
        # Devices depend on Wafer_id; we only set Name now, Wafer on first use.
        for d in devices_u:
            # We'll let per-row ensure Wafer_id and cache final id.
            pass
        for t in trt_u:
            # Same as devices: depends on Device_id & TrtType_id, set on first use
            pass

        # 4) Build DC/AC existence sets in batches (optional but recommended)
        #    Gather all (Trt, sMeasDate) seen in the df for DC/AC
        def _safe_dt(dt):
            try:
                return dt.strftime("%Y-%m-%d %H:%M:%S")
            except Exception:
                return None

        dc_keys = []
        ac_keys = []
        rows_tmp = []
        for r in dfUpload.itertuples(index=False):
            # r has attributes with column names (dots/space not allowed; your df seems fine)
            if getattr(r, 'DCVals', None) is not None:
                sdt = _safe_dt(getattr(r.DCVals, 'DateTime', r.DCVals.get('DateTime')))
                if sdt:
                    dc_keys.append((r.Trt, sdt))
            if getattr(r, 'ACVals', None) is not None:
                sdt = _safe_dt(getattr(r.ACVals, 'DateTime', r.ACVals.get('DateTime')))
                if sdt:
                    ac_keys.append((r.Trt, sdt))

        # Resolve Trt names -> Trt_id once per unique name with rich fields at first use
        # We'll defer the exact id creation to the row loop (to attach Device/Wafer/TrtType).

        # Helper to batch query existence from DB
        def _fill_exists(table, keys):
            """
            keys are (TrtName, sMeasDate). We resolve Trt_id lazily per chunk,
            then query existence in one go per chunk to populate cache['*_exists'] and ids.
            """
            if not keys:
                return {}

            ids_map = {}  # (Trt_id, sMeasDate) -> id{table}
            # chunk by chunk
            for i in range(0, len(keys), chunk_size_exists):
                sub = keys[i:i + chunk_size_exists]
                # Resolve TrtName -> Trt_id for this sub-batch
                trtname_to_id = {}
                for trt_name, _sdt in sub:
                    if trt_name in cache['Trts']:
                        trt_id = cache['Trts'][trt_name]
                    else:
                        # we can't create yet because we need Device/Wafer/TrtType to be correct
                        # So we only check existing by name if your schema allows it via join.
                        # Fallback: skip prefetch if we can't ensure proper id now.
                        trt_id = None
                    trtname_to_id[trt_name] = trt_id

                # Build Conditions OR chain: if trt_id is None we skip (we will insert new)
                conds = []
                vals = []
                for trt_name, sdt in sub:
                    tid = trtname_to_id.get(trt_name)
                    if tid is not None:
                        conds.append("(Trt_id = %s AND MeasDate = %s)")
                        vals.extend([tid, sdt])

                if not conds:
                    continue

                q = f"SELECT id{table}, Trt_id, MeasDate FROM {table} WHERE " + " OR ".join(conds)
                res = self._execute(q, tuple(vals))
                if res:
                    id_field = f"id{table}"
                    for row in res:
                        k = (int(row['Trt_id']), row['MeasDate'])
                        ids_map[k] = int(row[id_field])

            # Populate the cache sets
            if table == 'DCcharacts':
                cache['dc_exists'].update(ids_map.keys())
            else:
                cache['ac_exists'].update(ids_map.keys())
            return ids_map

        # Try prefill existence (will only fill for already-known Trt_ids)
        dc_ids_prefilled = _fill_exists('DCcharacts', dc_keys)
        ac_ids_prefilled = _fill_exists('ACcharacts', ac_keys)

        # 5) Row loop (minimal prints)
        n = len(dfUpload)
        for i, r in enumerate(dfUpload.itertuples(index=False)):
            if i % 50 == 0 or i == n - 1:
                print(f"Uploading {i + 1}/{n}")

            # --- Gate (cache by device + GateDate or filename) ---
            GateId = None
            GateData = getattr(r, 'GateData', None)
            if GateData is not None:
                # Build a stable key; prefer time if present, else filename
                try:
                    gate_dt = GateData['DateTime'].strftime("%Y-%m-%d %H:%M:%S")
                except Exception:
                    gate_dt = None
                gate_key = (r.Device, gate_dt or getattr(r, 'FileName', None))
                if gate_key in cache['Gates']:
                    GateId = cache['Gates'][gate_key]
                else:
                    GateFields = {'User': r.User, 'Name': f"{r.Device}-Gate", 'FileName': getattr(r, 'FileName', None)}
                    GateId = self.InsertGateCharact(GateData=GateData, Fields=GateFields)
                    cache['Gates'][gate_key] = GateId

            # --- FK ids (cached) ---
            Author_id = _getid_cached('Users', r.User, newvals={'Name': r.User})
            Wafer_id = _getid_cached('Wafers', r.Wafer, newvals={'Name': r.Wafer})

            # Device depends on Wafer; ensure proper creation once
            if r.Device in cache['Devices']:
                Device_id = cache['Devices'][r.Device]
            else:
                Device_id = self.GetId(Table='Devices', Value=r.Device,
                                       NewVals={'Name': r.Device, 'Wafer_id': Wafer_id})
                cache['Devices'][r.Device] = Device_id

            TrtType_id = _getid_cached('TrtTypes', r.TrtType,
                                       newvals=getattr(r, 'TrtTypeFields', None) or {'Name': r.TrtType})

            if r.Trt in cache['Trts']:
                Trt_id = cache['Trts'][r.Trt]
            else:
                Trt_id = self.GetId(Table='Trts', Value=r.Trt,
                                    NewVals={'Name': r.Trt, 'Device_id': Device_id, 'Type_id': TrtType_id})
                cache['Trts'][r.Trt] = Trt_id

            # --- Build minimal Fields dict quickly ---
            Fields = {'User': r.User, 'Wafer': r.Wafer, 'Device': r.Device, 'TrtType': r.TrtType, 'Trt': r.Trt,
                      'Gate_id': GateId}

            # --- Optional fields dict (avoid building huge dicts) ---
            # Extract only once, excluding NonOptFields
            # r._asdict() is available on namedtuples from itertuples(index=False, name=None?) – set name='Row' to use _asdict
            # Here we use getattr to avoid conversion cost:
            OptFields = {}
            for col in dfUpload.columns:
                if col not in NonOptFields:
                    OptFields[col] = getattr(r, col)

            # --- Insert DC/AC using your InsertCharact (keeps logic), but we help existence via cache ---
            # We can’t pass cached existence directly to InsertCharact without changing it;
            # however InsertCharact will now likely hit cached IDs and fewer selects.
            self.InsertCharact(DCVals=getattr(r, 'DCVals', None),
                               Fields=Fields,
                               ACVals=getattr(r, 'ACVals', None),
                               OptFields=OptFields,
                               TrtTypeFields=getattr(r, 'TrtTypeFields', None),
                               OverWrite=True)

def GetSerie(Data):
    pdser = {}
    try:
        d = DataCharAC(Data)
    except Exception as error:
        print('Error in GetSerie ', Data['Name'], ' ', Data['DateTime'])
        print(error)
        return None
    pdser['Name'] = d.Name
    if d.Name.find('Col') > 1:
        col = d.Name.split('-')[-1]
        pdser['Column'] = col
        row = d.Name.split('-')[-2].replace('Ch', 'Row')
        pdser['Row'] = row
    pdser['CharCl'] = d
    pdser['IsOk'] = d.IsOK
    pdser['ChName'] = d.ChName
    pdser['Date'] = d.GetDateTime()
    for k, v in d['Wafers'].items():
        if k == 'Name':
            pdser['Wafer'] = v
        else:
            pdser['Waf' + k] = v
    for k, v in d['Devices'].items():
        if k == 'Name':
            pdser['Device'] = v
        else:
            pdser['Dev' + k] = v
    for k, v in d['TrtTypes'].items():
        if k == 'Name':
            pdser['TrtType'] = v
        else:
            pdser[k] = v
    if ('Length' in pdser) and ('Width' in pdser):
        pdser['WL'] = float(pdser['Width']) / float(pdser['Length'])
    for k, v in d['Info'].items():
        if k == 'Gate_id':
            continue
        pdser[k] = v

    for cvk in ('CVCycle', 'CVSing', 'CVShape', 'CVSlewRate', 'CVFrequency'):
        pdser[cvk] = d.Get(cvk)

    return pd.Series(pdser)


def Data2Pandas(Data, Threads=True):
    if Threads:
        print("Converting {} Records".format(len(Data)))
        with Pool() as p:
            pdSeries = p.map(GetSerie, Data)
    else:
        pdSeries = []
        for ic, dd in enumerate(Data):
            print("Converting {} of {} -- {}".format(ic, len(Data), dd['Name']))
            pdSeries.append(GetSerie(Data=dd))

    dfRaw = pd.concat(pdSeries, axis=1).transpose()
    DataTypes = {}
    for col in dfRaw.keys():
        if col in ('Width', 'Length', 'WL', 'Pass', 'Area', 'Ph',
                   'IonStrength', 'AnalyteCon', 'CVSlewRate', 'CVFrequency','CVCycle'):
            dfRaw[col] = pd.to_numeric(dfRaw[col], errors='coerce')
            DataTypes[col] = float
        else:
            DataTypes[col] = pd.StringDtype()

    DataTypes['CharCl'] = object
    DataTypes['IsOk'] = bool
    DataTypes['Date'] = 'datetime64[ns]'

    return dfRaw.astype(DataTypes, errors='ignore')

if __name__ == '__main__':
    pass

#     # f = Fernet(open('key.key', 'rb').read())
#     # da = f.decrypt(open('Connection', 'rb').read())
#     # connection = pickle.loads(da)
#     MyDb = PyFETdb(open('key.key', 'rb').read())

# %% Test insert bin data
# OptFields = {}
# OptFields['Solution'] = 'TestWS'
# OptFields['FuncStep'] = 'TestWS'
# OptFields['Ph'] = 7.21

# by = pickle.dumps(OptFields,0)

# data = {}
# data['table'] = "DCcharacts"
# data['fields'] = {}
# data['fields'][0] = {}
# data['fields'][1] = {}
# data['fields'][2] = {}
# data['fields'][0]['name'] = "Trt_id"
# data['fields'][0]['value'] = 100
# data['fields'][0]['type'] = "int"
# data['fields'][1]['name'] = "Comments"
# data['fields'][1]['value'] = 'TestWS'
# data['fields'][1]['type'] = "str"
# data['fields'][2]['name'] = "Data"
# data['fields'][2]['value'] = by
# data['fields'][2]['type'] = "bin"

# res = ws_json.call(connection, data,"insert")
# '''
#  	Per un update, per exemple,
# data['where'] = "idDCcharacts=458586"
# res = ws_json_class.ws_json.call("pyfet", data,"update")
# '''
# try:
#  	error = res['error']
#  	if (error):
#          print("S'ha produït un error ")
#          print("\t" + str(res['errorCode']) + " : " + res['message'])
#  	else:
#          print("El resultat és: ")
#          print(res['result'])
# except:
#  	print("S'ha produït un error inesperat")
#  	sys.exit(0)


# qt = "SELECT * FROM DCcharacts WHERE Comments='TestWS'"

# Res = ws_json.call(connection, qt)


# %% Test Read
# DevicesList = ('B12744W3-Xip6NS', 'B12744W3-Xip5NS',
#                 )

# CharTable = 'DCcharacts'
# Conditions = {'Devices.name = ': DevicesList,
#               }

# GroupBase = {'Conditions': Conditions,
#               'Table': CharTable,
#               'Last': True,
#               'GetGate': True,
#               }


# res = MyDb.GetData2(**GroupBase)
# dfRaw = Data2Pandas(res)

# %% Test Read

# Fields = {}
# Fields['User'] = 'SB'

# OptFields = {}
# OptFields['Solution'] = 'TestWS'
# OptFields['FuncStep'] = 'TestWS'
# OptFields['Ph'] = 7.21
# OptFields['AnalyteCon'] = 'tt'
# OptFields['IonStrength'] = 10e-3
# OptFields['Comments'] = 'tt'

# Fields['Wafer'] = 'TestWS'
# Fields['Device'] = 'TestWS-Dev'
# Fields['Trt'] = 'TestWS-Dev'
# Fields['TrtType'] = 'RW50L50P3p0CS'
# Fields['Gate_id'] = None

# DCd = res[0]

# MyDb.InsertCharact(DCVals=DCd,
#                       ACVals=None,
#                       Fields=Fields,
#                       OptFields=OptFields,
#                       TrtTypeFields=None)
