"""
MigMan ETL Extract Module.

This module handles the extraction phase of the MigMan ETL pipeline.
It provides functionality to extract data from MigMan systems and
prepare it for the transformation phase.

The extraction process:
1. Connects to the MigMan system using configured credentials
2. Iterates through configured tables and extracts data
3. Handles inactive tables by skipping them
4. Uses ETLFileHandler for data persistence
5. Provides comprehensive logging throughout the process

Classes:
    MigManExtract: Main class handling MigMan data extraction.
"""

from importlib import resources
import logging
from pathlib import Path
import re
import subprocess
import tempfile
from typing import Union
import gzip
import shutil
import json
from nemo_library_etl.adapter._utils.db_handler_local import ETLDuckDBHandler
from nemo_library_etl.adapter._utils.db_handler_source import DatabaseHandlerSource
from nemo_library_etl.adapter._utils.dbandfileutils import _output_path
from nemo_library_etl.adapter.migman.config_models_migman import ConfigMigMan
from nemo_library_etl.adapter._utils.enums import ETLStep
from nemo_library_etl.adapter._utils.file_handler import ETLFileHandler
from nemo_library import NemoLibrary

from nemo_library_etl.adapter.migman.enums import MigManExtractAdapter
from nemo_library_etl.adapter.migman.migmanutils import MigManUtils


class MigManExtractFromDatabase:
    """
    Handles extraction of data from MigMan system.

    This class manages the extraction phase of the MigMan ETL pipeline,
    providing methods to connect to MigMan systems, retrieve data,
    and prepare it for subsequent transformation and loading phases.

    The extractor:
    - Uses NemoLibrary for core functionality and configuration
    - Integrates with Prefect logging for pipeline visibility
    - Processes tables based on configuration settings
    - Handles both active and inactive table configurations
    - Leverages ETLFileHandler for data persistence

    Attributes:
        nl (NemoLibrary): Core Nemo library instance for system integration.
        config: Configuration object from the Nemo library.
        logger: Prefect logger for pipeline execution tracking.
        cfg (PipelineMigMan): Pipeline configuration with extraction settings.
    """

    def __init__(
        self,
        nl: NemoLibrary,
        cfg: ConfigMigMan,
        logger: Union[logging.Logger, object],
        fh: ETLFileHandler,
        local_database: ETLDuckDBHandler,
    ) -> None:
        """
        Initialize the MigMan Extract instance.

        Sets up the extractor with the necessary library instances, configuration,
        and logging capabilities for the extraction process.

        Args:
            nl (NemoLibrary): Core Nemo library instance for system integration.
            cfg (PipelineZentis): Pipeline configuration object containing
                                                          extraction settings and rules.
            logger (Union[logging.Logger, object]): Logger instance for recording execution details.
                                                   Can be a standard Python logger or Prefect logger.
        """
        self.nl = nl
        self.cfg = cfg
        self.logger = logger
        self.fh = fh
        self.local_database = local_database

        super().__init__()

    def extract_from_database(self) -> None:
        """
        Execute the main extraction process for MigMan data.

        This method orchestrates the complete extraction process by:
        1. Logging the start of extraction
        2. Iterating through configured tables
        3. Skipping inactive tables
        4. Processing active tables and extracting their data
        5. Using ETLFileHandler for data persistence

        The method respects table activation settings and provides detailed
        logging for monitoring and debugging purposes.

        Note:
            The actual data extraction logic needs to be implemented based on
            the specific MigMan system requirements.
        """
        self.logger.info("Extracting all MigMan objects")

        # extract objects
        if not self.cfg.extract.debug_just_ingest_files_no_extraction:
            dbsrc = DatabaseHandlerSource(
                nl=self.nl,
                cfg=self.cfg,
                logger=self.logger,
                fh=self.fh,
            )

            if not self.cfg.setup.source_adapter:
                raise ValueError("No adapter specified for extraction")
        conn = None
        try:
            self.logger.info(
                f"Starting extraction for {self.cfg.setup.source_adapter} ..."
            )

            # identify tables to extract
            tables_to_extract, fields_to_extract = self.get_tables_and_fields(
                self.cfg.setup.source_adapter
            )

            if not tables_to_extract:
                raise ValueError("No tables to extract found")
            if not fields_to_extract:
                raise ValueError("No fields to extract found")

            self.logger.info(
                f"Tables to extract for {self.cfg.setup.source_adapter}: {tables_to_extract}"
            )
            self.logger.info(
                f"Fields to extract for {self.cfg.setup.source_adapter}: {fields_to_extract}"
            )

            if not self.cfg.extract.debug_just_ingest_files_no_extraction:

                # connect source database
                match self.cfg.setup.source_adapter:
                    case MigManExtractAdapter.GENERICODBC.value:
                        conn = dbsrc.connect_odbc(
                            odbc_connstr=self.cfg.extract.genericodbc.odbc_connstr,
                            timeout=self.cfg.extract.genericodbc.timeout,
                        )

                    case MigManExtractAdapter.INFORCOM.value:
                        conn = dbsrc.connect_odbc(
                            odbc_connstr=self.cfg.extract.inforcom.odbc_connstr,
                            timeout=self.cfg.extract.inforcom.timeout,
                        )
                    case MigManExtractAdapter.SAPECC.value:
                        conn = dbsrc.connect_hdb(
                            address=self.cfg.extract.sapecc.address,
                            port=self.cfg.extract.sapecc.port,
                            user=self.cfg.extract.sapecc.user,
                            password=self.cfg.extract.sapecc.password,
                            autocommit=self.cfg.extract.sapecc.autocommit,
                        )
                    case MigManExtractAdapter.SAPTENANT.value:
                        conn = dbsrc.connect_hdb(
                            address=self.cfg.extract.saptenant.address,
                            port=self.cfg.extract.saptenant.port,
                            user=self.cfg.extract.saptenant.user,
                            password=self.cfg.extract.saptenant.password,
                            autocommit=self.cfg.extract.saptenant.autocommit,
                        )
                    case MigManExtractAdapter.SAPSYSTEMDB.value:
                        conn = dbsrc.connect_hdb(
                            address=self.cfg.extract.sapsystemdb.address,
                            port=self.cfg.extract.sapsystemdb.port,
                            user=self.cfg.extract.sapsystemdb.user,
                            password=self.cfg.extract.sapsystemdb.password,
                            database=self.cfg.extract.sapsystemdb.database,
                            autocommit=self.cfg.extract.sapsystemdb.autocommit,
                        )
                    case MigManExtractAdapter.PROALPHA.value:
                        conn = dbsrc.connect_odbc(
                            odbc_connstr=self.cfg.extract.proalpha.odbc_connstr,
                            timeout=self.cfg.extract.proalpha.timeout,
                        )
                    case MigManExtractAdapter.INFORAS.value:
                        conn = dbsrc.connect_odbc(
                            odbc_connstr=self.cfg.extract.inforas.odbc_connstr,
                            timeout=self.cfg.extract.inforas.timeout,
                        )

                    case MigManExtractAdapter.SAGEKHK.value:
                        conn = dbsrc.connect_odbc(
                            odbc_connstr=self.cfg.extract.sagekhk.odbc_connstr,
                            timeout=self.cfg.extract.sagekhk.timeout,
                        )
                    case _:
                        raise ValueError(
                            f"Unknown adapter: {self.cfg.setup.source_adapter}"
                        )

                if not conn:
                    raise ConnectionError(
                        f"Failed to connect to {self.cfg.setup.source_adapter} database"
                    )

            # extract tables
            for table_name in tables_to_extract:

                # -------------------------------------------------------------------------
                # Workaround for INFORCOM.RELTEXT: ODBC driver cannot fetch LONG/CLOB/RTF.
                # We perform a SQLPlus-based export instead, then ingest the CSV into DuckDB.
                # Afterwards, the CSV is compressed to GZIP and the plain file is removed.
                # -------------------------------------------------------------------------

                if (
                    self.cfg.setup.source_adapter == MigManExtractAdapter.INFORCOM.value
                    and table_name.lower() == "reltext"
                    and self.cfg.extract.inforcom.odbc_bug_workaround_for_reltext_active
                ):
                    self.logger.warning(
                        "Using SQLPlus workaround for INFORCOM.RELTEXT (ODBC LOB bug)."
                    )

                    # ---------------------------------------------------------------------
                    # 1. Determine CSV output path and ensure directory exists.
                    # ---------------------------------------------------------------------
                    csv_path = _output_path(
                        etl_directory=self.cfg.etl_directory,
                        step=ETLStep.EXTRACT,
                        substep=None,
                        entity=table_name,
                        filename=None,
                        suffix=".csv",
                    )
                    csv_path.parent.mkdir(parents=True, exist_ok=True)
                    gz_path = csv_path.with_suffix(csv_path.suffix + ".gz")

                    self.logger.info(f"CSV output path will be: {csv_path}")

                    # ---------------------------------------------------------------------
                    # 2. Load the SQLPlus script template and inject the SPOOL path.
                    # ---------------------------------------------------------------------
                    sql_template_path = (
                        resources.files("nemo_library_etl")
                        / "adapter"
                        / "migman"
                        / "config"
                        / "extrakt"
                        / "inforcom"
                        / "reltext"
                        / "export_reltext.sql"
                    )
                    template_content = sql_template_path.read_text(encoding="utf-8")

                    # SQLPlus prefers Windows-style paths with backslashes and quotes
                    spool_path_literal = f"\"{str(csv_path.resolve())}\""
                    rendered_sql = template_content.replace("__SPOOL_PATH__", spool_path_literal)

                    self.logger.debug("Rendered SQLPlus script:\n" + rendered_sql)

                    # ---------------------------------------------------------------------
                    # 3. Write the rendered SQL script to a temporary file.
                    # ---------------------------------------------------------------------
                    with tempfile.NamedTemporaryFile(
                        "w", delete=False, suffix=".sql", encoding="utf-8"
                    ) as tmp_sql:
                        tmp_sql.write(rendered_sql)
                        tmp_sql_path = Path(tmp_sql.name)

                    self.logger.info(f"Temporary SQL script created at: {tmp_sql_path}")

                    # ---------------------------------------------------------------------
                    # 4. Build SQLPlus command.
                    #    All config values exist because we use Pydantic (no getattr needed).
                    # ---------------------------------------------------------------------
                    sqlplus_exe = self.cfg.extract.inforcom.odbc_bug_workaround_for_reltext_sqlplus_exe
                    sqlplus_connect = self.cfg.extract.inforcom.odbc_bug_workaround_for_reltext_sqlplus_connect
                    sqlplus_logfile = (
                        Path(self.cfg.etl_directory)
                        / self.cfg.extract.inforcom.odbc_bug_workaround_for_reltext_sqlplus_logfile
                    )
                    silent = self.cfg.extract.inforcom.odbc_bug_workaround_for_reltext_sqlplus_silent

                    cmd = [sqlplus_exe]
                    if silent:
                        cmd.append("-S")
                    cmd.extend([sqlplus_connect, f"@{tmp_sql_path}"])

                    self.logger.info(f"Executing SQLPlus: {cmd}")

                    # ---------------------------------------------------------------------
                    # 5. Execute SQLPlus and capture output.
                    #    On success: minimal log.
                    #    On failure: full STDOUT/STDERR in log.
                    # ---------------------------------------------------------------------

                    if not self.cfg.extract.debug_just_ingest_files_no_extraction:                        
                        result = subprocess.run(
                            cmd,
                            capture_output=True,
                            text=True,
                            shell=False,
                        )

                        if result.returncode != 0:
                            self.logger.error("SQLPlus failed during RELTEXT export.")
                            self.logger.error("SQLPlus STDOUT:\n" + result.stdout)
                            self.logger.error("SQLPlus STDERR:\n" + result.stderr)
                            raise RuntimeError("SQLPlus export failed. See log above.")

                        self.logger.info("SQLPlus completed successfully for RELTEXT.")
                        
                        # Write stdout/stderr to a SQLPlus log for later inspection
                        sqlplus_logfile.write_text(
                            result.stdout + "\n" + result.stderr,
                            encoding="utf-8"
                        )
                        self.logger.info(f"SQLPlus log written to: {sqlplus_logfile}")

                        # ---------------------------------------------------------------------
                        # 6. Delete temp SQL file (best-effort).
                        # ---------------------------------------------------------------------
                        try:
                            tmp_sql_path.unlink()
                        except Exception as cleanup_err:
                            self.logger.warning(
                                f"Could not delete temporary SQL script {tmp_sql_path}: {cleanup_err}"
                            )

                        # ---------------------------------------------------------------------
                        # 7. Compress the CSV to GZIP, removing duplicate header lines, and
                        #    then remove the plain CSV file.
                        # ---------------------------------------------------------------------
                        gz_path = csv_path.with_suffix(csv_path.suffix + ".gz")
                        self.logger.info(
                            f"Compressing RELTEXT CSV to GZIP (deduplicating headers): {csv_path} -> {gz_path}"
                        )

                        try:
                            # Work on binary level to avoid encoding issues.
                            with open(csv_path, "rb") as f_in, gzip.open(gz_path, "wb") as f_out:
                                header_line = None
                                for line in f_in:
                                    if header_line is None:
                                        # First line is the header; always write it.
                                        header_line = line
                                        f_out.write(line)
                                    else:
                                        # Skip any line that is exactly the same as the header line.
                                        if line == header_line:
                                            continue
                                        f_out.write(line)

                            csv_path.unlink()
                            self.logger.info(
                                f"CSV compressed with header deduplication; original file deleted. Using {gz_path} for ingestion."
                            )
                        except Exception as compress_err:
                            self.logger.error(
                                f"Failed to compress RELTEXT CSV to GZIP: {compress_err}"
                            )
                            raise
                        


                    # ---------------------------------------------------------------------
                    # 9. Load the GZIP'ed CSV exported by SQLPlus into DuckDB.
                    #    DuckDB read_csv_auto() can transparently read .gz files.
                    # ---------------------------------------------------------------------
                    self.logger.info(f"Ingesting RELTEXT CSV (GZIP) from: {gz_path}")

                    self.local_database.ingest_csv(
                        filename=gz_path,
                        table_name=table_name,
                        create_mode="replace",
                        header=True,
                        separator=";",         # matches DELIMITER ';'
                        auto_detect=True,      # wichtig für das Schema
                    )
                    self.logger.info("CSV (GZIP) successfully ingested into DuckDB for RELTEXT.")

                    # -------------------------------------------------------------------------
                    # Workaround for INFORCOM.RELTEXT: ODBC driver cannot fetch LONG/CLOB/RTF.
                    # We perform a SQLPlus-based export instead, then ingest the CSV into DuckDB.
                    # Afterwards, the CSV is compressed to GZIP and the plain file is removed.
                    # -------------------------------------------------------------------------

                else:
                    self.logger.info(
                        f"Extracting table {table_name} from {self.cfg.setup.source_adapter} ..."
                    )
                    source_table_name = table_name

                    if self.cfg.setup.source_adapter == MigManExtractAdapter.INFORAS.value:
                        source_table_name = self._resolve_inforas_source_table(
                            conn=conn,
                            table_name=table_name,
                        )
                    else:
                        source_table_name = (
                            f"{getattr(self.cfg.extract, f'{self.cfg.setup.source_adapter}').table_prefix}"
                            f"{table_name}"
                        )

                    query = f"""
    SELECT
        {', '.join(fields_to_extract.get(table_name, []))}
    FROM
        {source_table_name}
    """
                    self.logger.info(f"Using query: {query}")
                    dbsrc.generic_odbc_extract(
                        conn=conn,
                        query=query,
                        entity=table_name,
                        step=ETLStep.EXTRACT,
                        chunksize=getattr(
                            self.cfg.extract, f"{self.cfg.setup.source_adapter}"
                        ).chunk_size,
                    )
                    self.logger.info(f"Finished extracting table {table_name}.")

                    self.logger.info(
                        f"Ingesting table {table_name} into local database..."
                    )
                    self.logger.info("Sanitizing JSONL keys (\\ -> _) before DuckDB ingest...")
                    self._sanitize_jsonl_files_for_entity(table_name)
                                        
                    self.local_database.ingest_jsonl(
                        step=ETLStep.EXTRACT,
                        entity=table_name,
                        ignore_nonexistent=True,
                        create_mode="replace",  # or "append"
                        table_name=table_name,  # keep your original table name
                        add_metadata=True,  # adds _source_path & _ingested_at
                    )

                if self.cfg.extract.load_to_nemo:
                    self.local_database.upload_table_to_nemo(
                        table_name=table_name,
                        project_name=f"{self.cfg.extract.nemo_project_prefix}{table_name}",
                        delete_temp_files=self.cfg.extract.delete_temp_files,
                    )


            self.logger.info(
                f"Completed extraction for {self.cfg.setup.source_adapter}."
            )
        except Exception as e:
            self.logger.error(
                f"Error during extraction for {self.cfg.setup.source_adapter}: {e}"
            )
            raise
        finally:
            try:
                if conn:
                    conn.cursor().close()
                    conn.close()
            except Exception as close_err:
                self.logger.warning(
                    f"Failed to close connection for {self.cfg.setup.source_adapter}: {close_err}"
                )
    def _normalize_inforas_table_prefixes(self) -> list[str]:
        prefixes = self.cfg.extract.inforas.table_prefixes or []
        normalized = []

        for prefix in prefixes:
            cleaned = (prefix or "").strip().rstrip(".")
            if cleaned:
                normalized.append(f"{cleaned}.")

        # Reihenfolge beibehalten, Dubletten entfernen
        return list(dict.fromkeys(normalized))

    def _inforas_table_exists(self, conn, source_table_name: str) -> bool:
        cursor = conn.cursor()
        try:
            cursor.execute(f"SELECT * FROM {source_table_name} WHERE 1 = 0")
            return True
        except Exception as exc:
            self.logger.debug(
                f"INFORAS schema candidate not usable: {source_table_name} ({exc})"
            )
            return False
        finally:
            try:
                cursor.close()
            except Exception:
                pass

    def _resolve_inforas_source_table(self, conn, table_name: str) -> str:
        prefixes = self._normalize_inforas_table_prefixes()

        if not prefixes:
            raise ValueError(
                "INFORAS requires at least one entry in extract.inforas.table_prefixes"
            )

        for prefix in prefixes:
            source_table_name = f"{prefix}{table_name}"
            if self._inforas_table_exists(conn, source_table_name):
                self.logger.info(
                    f"Resolved INFORAS table {table_name} to source table {source_table_name}"
                )
                return source_table_name

        raise ValueError(
            f"Could not resolve INFORAS table '{table_name}' in any configured schema prefix: {prefixes}"
        )
    def get_tables_and_fields(self, adapter) -> tuple[list[str], dict[str, list[str]]]:

        tables_to_extract = []
        fields_to_extract = {}

        if getattr(self.cfg.extract, f"{adapter}").table_selector == "all":
            tables_to_extract = getattr(self.cfg.extract, f"{adapter}").tables
            fields_to_extract = {table: ["*"] for table in tables_to_extract}
        elif getattr(self.cfg.extract, f"{adapter}").table_selector == "join_parser":
            for project in MigManUtils.get_migman_projects(self.cfg):

                query = MigManUtils.getJoinQuery(adapter=adapter, project=project)

                tables_to_extract.extend(self.local_database.extract_tables(sql=query))
                fields_in_query = self.local_database.extract_fields_by_base_table(
                    sql=query
                )
                for table, fields in fields_in_query.items():
                    fields_to_extract.setdefault(table, []).extend(fields)

        tables_to_extract = list(set(tables_to_extract))  # deduplicate
        return tables_to_extract, fields_to_extract
    
    def _sanitize_json_keys(self, obj):
        """
        Recursively sanitize dict keys:
        - '\' -> '_'
        - '/' -> '_'
        """
        if isinstance(obj, dict):
            new_obj = {}
            for k, v in obj.items():
                if isinstance(k, str):
                    new_key = k.replace("\\", "_").replace("/", "_")
                else:
                    new_key = k
                new_obj[new_key] = self._sanitize_json_keys(v)
            return new_obj

        if isinstance(obj, list):
            return [self._sanitize_json_keys(x) for x in obj]

        return obj


    def _sanitize_jsonl_files_for_entity(self, entity: str) -> None:
        """
        Finds JSONL/JSONL.GZ files for the given entity under the ETL directory
        and rewrites them deterministically:
        - keys sanitized (\\ and / -> _)
        - output filename is ALWAYS lowercase
        - output is ALWAYS overwritten by this process
        - original (non-lowercase) file is removed if different
        """
        base = Path(self.cfg.etl_directory)

        # Try common layouts. Adjust if needed, but this covers your log path etl\migman\extract\...
        extract_candidates = [
            base / "extract",
            base / "migman" / "extract",
        ]
        extract_dir = next((p for p in extract_candidates if p.exists()), None)

        if extract_dir is None:
            # fallback: search from base
            search_roots = [base]
        else:
            search_roots = [extract_dir]

        # 1) Prefer exact expected filenames in extract_dir
        candidates: list[Path] = []
        if extract_dir is not None:
            expected = [
                extract_dir / f"{entity.lower()}.jsonl",
                extract_dir / f"{entity.lower()}.jsonl.gz",
                extract_dir / f"{entity}.jsonl",
                extract_dir / f"{entity}.jsonl.gz",
            ]
            candidates = [p for p in expected if p.exists()]

        # 2) Fallback: glob (case-insensitive-ish via multiple patterns)
        if not candidates:
            for root in search_roots:
                candidates.extend(root.rglob(f"*{entity}*.jsonl"))
                candidates.extend(root.rglob(f"*{entity}*.jsonl.gz"))
                candidates.extend(root.rglob(f"*{entity.lower()}*.jsonl"))
                candidates.extend(root.rglob(f"*{entity.lower()}*.jsonl.gz"))

        # Deduplicate candidates
        uniq = {}
        for p in candidates:
            uniq[str(p.resolve())] = p
        candidates = list(uniq.values())

        for p in candidates:
            self.logger.info(f"Sanitizer will process: {p}")

        skipped = 0

        for path in candidates:
            lower_path = path.with_name(path.name.lower())

            # -------------------------
            # Case 1: *.jsonl.gz
            # -------------------------
            if path.suffixes[-2:] == [".jsonl", ".gz"]:
                tmp_plain = lower_path.with_suffix(".jsonl.tmp")

                # Read gz -> sanitize -> write tmp plain
                with gzip.open(path, "rt", encoding="utf-8") as fin, open(tmp_plain, "wt", encoding="utf-8") as fout:
                    for line in fin:
                        line = line.strip()
                        if not line:
                            continue
                        obj = json.loads(line)
                        if not isinstance(obj, dict):
                            skipped += 1
                            continue
                        obj = self._sanitize_json_keys(obj)
                        fout.write(json.dumps(obj, ensure_ascii=False) + "\n")

                # Write tmp plain -> lower_path gz (overwrite)
                with open(tmp_plain, "rt", encoding="utf-8") as fin2, gzip.open(lower_path, "wt", encoding="utf-8") as fout2:
                    for line in fin2:
                        fout2.write(line)

                tmp_plain.unlink(missing_ok=True)

                # Remove original if different
                if path != lower_path and path.exists():
                    try:
                        path.unlink()
                    except Exception as e:
                        self.logger.warning(f"Could not delete original file {path}: {e}")

                self.logger.info(f"Sanitized and overwritten: {lower_path}")

            # -------------------------
            # Case 2: *.jsonl (plain)
            # -------------------------
            else:
                tmp = lower_path.with_suffix(lower_path.suffix + ".tmp")

                # Read -> sanitize -> write tmp
                with open(path, "rt", encoding="utf-8") as fin, open(tmp, "wt", encoding="utf-8") as fout:
                    for line in fin:
                        line = line.strip()
                        if not line:
                            continue
                        obj = json.loads(line)
                        if not isinstance(obj, dict):
                            skipped += 1
                            continue
                        obj = self._sanitize_json_keys(obj)
                        fout.write(json.dumps(obj, ensure_ascii=False) + "\n")

                # Overwrite lower_path
                tmp.replace(lower_path)

                # Remove original if different
                if path != lower_path and path.exists():
                    try:
                        path.unlink()
                    except Exception as e:
                        self.logger.warning(f"Could not delete original file {path}: {e}")

                self.logger.info(f"Sanitized and overwritten: {lower_path}")

        if candidates:
            self.logger.info(
                f"Sanitized JSONL keys (\\ and / -> _) for entity '{entity}' in {len(candidates)} file(s)."
            )
        else:
            self.logger.warning(f"No JSONL files found to sanitize for entity '{entity}'.")
