"""
MigMan ETL Load Module.

This module handles the loading phase of the MigMan ETL pipeline.
It takes the transformed data and loads it into the target system, typically the
Nemo database or data warehouse.

The loading process typically includes:
1. Data validation before insertion
2. Connection management to target systems
3. Batch processing for efficient data loading
4. Error handling and rollback capabilities
5. Data integrity checks post-loading
6. Performance optimization for large datasets
7. Comprehensive logging throughout the process

Classes:
    MigManLoad: Main class handling MigMan data loading.
"""

import logging
from datetime import datetime
import html
import json
from functools import lru_cache
from importlib import metadata as importlib_metadata
import os
from pathlib import Path
import re
from typing import Union

from nemo_library_etl.adapter._utils.db_handler_local import ETLDuckDBHandler
from nemo_library_etl.adapter.migman.config_models_migman import ConfigMigMan
from nemo_library_etl.adapter._utils.file_handler import ETLFileHandler
from nemo_library import NemoLibrary
from nemo_library.model.project import Project
from nemo_library.model.column import Column
from nemo_library.model.report import Report
from nemo_library.model.rule import Rule

from nemo_library_etl.adapter.migman.enums import MigManTransformStep
from nemo_library_etl.adapter.migman.migmanutils import MigManUtils
from nemo_library.utils.utils import get_internal_name


DATA_CLASSIFICATIONS = {
    "S_Adresse.HomePage": "url",
    "S_Adresse.EMail": "email",
    "S_Adresse.Handy": "phone_number",
    "S_Adresse.Telefax": "phone_number",
    "S_Adresse.Telefon": "phone_number",
    "S_Adresse.Telefon2": "phone_number",
    "S_Adresse.Name1": "address",
    "S_Adresse.Name2": "address",
    "S_Adresse.Name3": "address",
    "S_Adresse.Staat": "address",
    "S_Adresse.Ort": "address",
    "S_Adresse.PLZ": "address",
    "S_Adresse.Strasse": "address",
    "S_Adresse.Hausnummer": "address",
}


def _quote_sql_identifier(value: str) -> str:
    escaped_value = value.replace('"', '""')
    return f'"{escaped_value}"'


def _beautify_report_column_name(value: str) -> str:
    sanitized = value.translate(
        str.maketrans(
            {
                ",": " ",
                ";": " ",
                "\r": " ",
                "\n": " ",
                "\t": " ",
            }
        )
    )
    return re.sub(r"\s+", " ", sanitized).strip()


def _parse_report_count(value: object) -> int:
    text = "" if value is None else str(value).strip()
    if not text:
        return 0
    try:
        return int(float(text))
    except ValueError:
        return 0


def _filter_customer_report_columns(df):
    special_columns = {"ERROR_CODE_CONCAT", "ERROR_CODE_COUNT_OVERALL", "RECORD_PIPE"}
    columns = list(df.columns)
    keep_columns: list[str] = []
    processed_columns: set[str] = set()

    for column in columns:
        if column in processed_columns:
            continue

        if column in special_columns:
            keep_columns.append(column)
            processed_columns.add(column)
            continue

        if column.endswith(" Count") and column[: -len(" Count")] in df.columns:
            flag_column = column[: -len(" Count")]
            count_column = column
        elif f"{column} Count" in df.columns:
            flag_column = column
            count_column = f"{column} Count"
        else:
            keep_columns.append(column)
            processed_columns.add(column)
            continue

        count_value = 0 if df.empty else max(_parse_report_count(value) for value in df[count_column])
        if count_value > 0:
            keep_columns.extend([flag_column, count_column])

        processed_columns.update({flag_column, count_column})

    return df.loc[:, [column for column in keep_columns if column in df.columns]]


@lru_cache(maxsize=1)
def _get_nemo_etl_version() -> str:
    setup_path = Path(__file__).resolve().parents[3] / "setup.py"
    if setup_path.exists():
        setup_content = setup_path.read_text(encoding="utf-8")
        match = re.search(r'version="([^"]+)"', setup_content)
        if match:
            return match.group(1)

    for dist_name in ("nemo_library_etl", "nemo-library-etl"):
        try:
            return importlib_metadata.version(dist_name)
        except importlib_metadata.PackageNotFoundError:
            continue

    try:
        from nemo_library_etl._version import __version__ as package_version

        return package_version
    except Exception:
        return "unknown"


def _relative_report_path(path: Path) -> Path:
    if not path.is_absolute():
        return path

    try:
        return Path(os.path.relpath(path, Path.cwd()))
    except ValueError:
        return Path(path.name)


def _create_deficiency_report_html(
    report_title: str,
    file_path: Path,
    created_at: datetime,
    df,
    error_row_count: int,
    generated_by_version: str,
) -> str:
    def _json_for_html_script(value: object) -> str:
        return (
            json.dumps(value, ensure_ascii=False)
            .replace("<", "\\u003c")
            .replace(">", "\\u003e")
            .replace("&", "\\u0026")
            .replace("\u2028", "\\u2028")
            .replace("\u2029", "\\u2029")
        )

    columns = [str(column) for column in df.columns]
    records = []

    for row in df.fillna("").astype(str).to_dict(orient="records"):
        records.append({str(key): str(value) for key, value in row.items()})

    columns_json = _json_for_html_script(columns)
    records_json = _json_for_html_script(records)
    report_title_html = html.escape(report_title)
    file_path_html = html.escape(file_path.as_posix())
    entry_count_html = html.escape(str(len(df.index)))
    error_row_count_html = html.escape(str(error_row_count))
    created_at_html = html.escape(created_at.strftime("%Y-%m-%d %H:%M:%S"))
    generated_by_version_html = html.escape(generated_by_version)

    return f"""<!DOCTYPE html>
<html lang="de">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>{report_title_html}</title>
    <style>
        :root {{
            --border: #d0d7de;
            --bg-header: #f6f8fa;
            --bg-panel: #ffffff;
            --bg-panel-muted: #f8fafc;
            --text: #1f2328;
            --text-muted: #57606a;
            --accent: #0969da;
            --shadow: 0 8px 24px rgba(140, 149, 159, 0.2);
        }}

        * {{
            box-sizing: border-box;
        }}

        body {{
            margin: 0;
            padding: 24px;
            font-family: Arial, Helvetica, sans-serif;
            color: var(--text);
            background: #ffffff;
        }}

        h1 {{
            margin: 0 0 12px 0;
            font-size: 28px;
        }}

        hr {{
            border: 0;
            border-top: 1px solid var(--border);
            margin: 16px 0 20px 0;
        }}

        .meta {{
            margin-bottom: 24px;
            padding: 12px 16px;
            border: 1px solid var(--border);
            border-radius: 8px;
            background: var(--bg-panel-muted);
        }}

        .meta-row {{
            display: flex;
            gap: 12px;
            margin: 6px 0;
            flex-wrap: wrap;
        }}

        .meta-label {{
            min-width: 120px;
            font-weight: 700;
        }}

        .toolbar {{
            display: flex;
            justify-content: space-between;
            align-items: center;
            gap: 12px;
            margin-bottom: 12px;
            flex-wrap: wrap;
        }}

        .toolbar-left {{
            display: flex;
            gap: 8px;
            align-items: center;
            flex-wrap: wrap;
        }}

        .button {{
            border: 1px solid var(--border);
            background: #fff;
            color: var(--text);
            padding: 8px 12px;
            border-radius: 6px;
            cursor: pointer;
            font-size: 14px;
        }}

        .button:hover {{
            border-color: var(--accent);
            color: var(--accent);
        }}

        .status {{
            color: var(--text-muted);
            font-size: 14px;
        }}

        .table-wrapper {{
            overflow: auto;
            border: 1px solid var(--border);
            border-radius: 8px;
        }}

        table {{
            width: 100%;
            border-collapse: collapse;
            min-width: 900px;
        }}

        thead th {{
            position: sticky;
            top: 0;
            z-index: 2;
            background: var(--bg-header);
            border-bottom: 1px solid var(--border);
            padding: 10px;
            text-align: left;
            vertical-align: top;
        }}

        tbody td {{
            padding: 10px;
            border-top: 1px solid var(--border);
            vertical-align: top;
            white-space: pre-wrap;
            word-break: break-word;
        }}

        tbody tr:nth-child(even) {{
            background: #fcfcfd;
        }}

        .header-cell {{
            display: flex;
            flex-direction: column;
            gap: 8px;
            min-width: 160px;
        }}

        .header-main {{
            display: flex;
            align-items: center;
            justify-content: space-between;
            gap: 8px;
        }}

        .sort-button {{
            border: 0;
            background: transparent;
            cursor: pointer;
            font-weight: 700;
            color: inherit;
            text-align: left;
            padding: 0;
            display: inline-flex;
            gap: 6px;
            align-items: center;
        }}

        .sort-indicator {{
            font-size: 12px;
            color: var(--text-muted);
        }}

        .filter-toggle {{
            align-self: flex-start;
            border: 1px solid var(--border);
            background: #fff;
            border-radius: 6px;
            padding: 5px 8px;
            cursor: pointer;
            font-size: 12px;
        }}

        .filter-panel {{
            display: none;
            position: absolute;
            margin-top: 6px;
            padding: 10px;
            background: var(--bg-panel);
            border: 1px solid var(--border);
            border-radius: 8px;
            box-shadow: var(--shadow);
            min-width: 260px;
            max-width: 360px;
            max-height: 320px;
            overflow: auto;
            z-index: 5;
        }}

        .filter-panel.open {{
            display: block;
        }}

        .filter-actions {{
            display: flex;
            gap: 8px;
            margin-bottom: 10px;
            flex-wrap: wrap;
        }}

        .filter-option {{
            display: flex;
            gap: 8px;
            align-items: flex-start;
            margin: 6px 0;
            font-weight: 400;
        }}

        .filter-option span {{
            word-break: break-word;
        }}

        .muted {{
            color: var(--text-muted);
        }}

        .empty-state {{
            padding: 16px;
            color: var(--text-muted);
        }}
    </style>
</head>
<body>
    <h1>{report_title_html}</h1>
    <hr>

    <div class="meta">
        <div class="meta-row"><span class="meta-label">Dateipfad</span><span>{file_path_html}</span></div>
        <div class="meta-row"><span class="meta-label">Anzahl Eintraege / Fehlerzeilen</span><span>{entry_count_html} / {error_row_count_html}</span></div>
        <div class="meta-row"><span class="meta-label">Erzeugt am</span><span>{created_at_html}</span></div>
        <div class="meta-row"><span class="meta-label">Erzeugt durch / Version</span><span>{generated_by_version_html}</span></div>
    </div>

    <div class="toolbar">
        <div class="toolbar-left">
            <button id="reset-sort" class="button" type="button">Sortierung zurücksetzen</button>
            <button id="reset-filter" class="button" type="button">Filter zurücksetzen</button>
        </div>
        <div id="table-status" class="status"></div>
    </div>

    <div class="table-wrapper">
        <table id="report-table">
            <thead></thead>
            <tbody></tbody>
        </table>
    </div>

    <script>
        const columns = {columns_json};
        const originalRows = {records_json};

        const table = document.getElementById("report-table");
        const thead = table.querySelector("thead");
        const tbody = table.querySelector("tbody");
        const statusNode = document.getElementById("table-status");
        const resetSortButton = document.getElementById("reset-sort");
        const resetFilterButton = document.getElementById("reset-filter");

        const sortState = {{ column: null, direction: null }};
        const filterState = {{}};

        function escapeHtml(value) {{
            return String(value)
                .replace(/&/g, "&amp;")
                .replace(/</g, "&lt;")
                .replace(/>/g, "&gt;")
                .replace(/"/g, "&quot;")
                .replace(/'/g, "&#39;");
        }}

        function normalizeValue(value) {{
            if (value === null || value === undefined) {{
                return "";
            }}
            return String(value);
        }}

        function compareValues(a, b) {{
            const aNumber = Number(a);
            const bNumber = Number(b);
            const aIsNumber = a !== "" && !Number.isNaN(aNumber);
            const bIsNumber = b !== "" && !Number.isNaN(bNumber);

            if (aIsNumber && bIsNumber) {{
                return aNumber - bNumber;
            }}

            return a.localeCompare(b, undefined, {{ numeric: true, sensitivity: "base" }});
        }}

        function getFilteredRows() {{
            return originalRows.filter((row) => {{
                return columns.every((column) => {{
                    const selectedValues = filterState[column];
                    if (!selectedValues) {{
                        return true;
                    }}
                    return selectedValues.has(normalizeValue(row[column]));
                }});
            }});
        }}

        function getVisibleRows() {{
            const rows = [...getFilteredRows()];

            if (sortState.column && sortState.direction) {{
                rows.sort((left, right) => {{
                    const leftValue = normalizeValue(left[sortState.column]);
                    const rightValue = normalizeValue(right[sortState.column]);
                    const result = compareValues(leftValue, rightValue);
                    return sortState.direction === "asc" ? result : -result;
                }});
            }}

            return rows;
        }}

        function cycleSort(column) {{
            if (sortState.column !== column) {{
                sortState.column = column;
                sortState.direction = "asc";
                render();
                return;
            }}

            if (sortState.direction === "asc") {{
                sortState.direction = "desc";
                render();
                return;
            }}

            if (sortState.direction === "desc") {{
                sortState.column = null;
                sortState.direction = null;
                render();
                return;
            }}

            sortState.column = column;
            sortState.direction = "asc";
            render();
        }}

        function getDistinctValues(targetColumn) {{
            const rows = originalRows.filter((row) => {{
                return columns.every((column) => {{
                    if (column === targetColumn) {{
                        return true;
                    }}

                    const selectedValues = filterState[column];
                    if (!selectedValues) {{
                        return true;
                    }}

                    return selectedValues.has(normalizeValue(row[column]));
                }});
            }});

            const distinct = new Set(rows.map((row) => normalizeValue(row[targetColumn])));
            return Array.from(distinct).sort((a, b) => compareValues(a, b));
        }}

        function updateStatus(rows) {{
            const activeFilters = Object.keys(filterState).length;
            const sortLabel = sortState.column
                ? `Sortierung: ${{sortState.column}} (${{sortState.direction === "asc" ? "aufsteigend" : "absteigend"}})`
                : "Sortierung: keine";
            statusNode.textContent = `${{rows.length}} von ${{originalRows.length}} Zeilen sichtbar | ${{sortLabel}} | aktive Filter: ${{activeFilters}}`;
        }}

        function renderHeader() {{
            const row = document.createElement("tr");

            columns.forEach((column) => {{
                const th = document.createElement("th");
                th.style.position = "relative";

                const wrapper = document.createElement("div");
                wrapper.className = "header-cell";

                const headerMain = document.createElement("div");
                headerMain.className = "header-main";

                const sortButton = document.createElement("button");
                sortButton.type = "button";
                sortButton.className = "sort-button";
                sortButton.addEventListener("click", () => cycleSort(column));

                const label = document.createElement("span");
                label.textContent = column;

                const indicator = document.createElement("span");
                indicator.className = "sort-indicator";
                if (sortState.column === column && sortState.direction === "asc") {{
                    indicator.textContent = "▲";
                }} else if (sortState.column === column && sortState.direction === "desc") {{
                    indicator.textContent = "▼";
                }} else {{
                    indicator.textContent = "↕";
                }}

                sortButton.appendChild(label);
                sortButton.appendChild(indicator);

                headerMain.appendChild(sortButton);

                const filterToggle = document.createElement("button");
                filterToggle.type = "button";
                filterToggle.className = "filter-toggle";
                filterToggle.textContent = "Filter anzeigen";

                const panel = document.createElement("div");
                panel.className = "filter-panel";

                filterToggle.addEventListener("click", (event) => {{
                    event.stopPropagation();
                    document.querySelectorAll(".filter-panel.open").forEach((node) => {{
                        if (node !== panel) {{
                            node.classList.remove("open");
                        }}
                    }});
                    panel.classList.toggle("open");
                }});

                const actions = document.createElement("div");
                actions.className = "filter-actions";

                const selectAllButton = document.createElement("button");
                selectAllButton.type = "button";
                selectAllButton.className = "button";
                selectAllButton.textContent = "Alle";
                selectAllButton.style.fontSize = "12px";
                selectAllButton.style.padding = "6px 8px";

                const clearButton = document.createElement("button");
                clearButton.type = "button";
                clearButton.className = "button";
                clearButton.textContent = "Keine";
                clearButton.style.fontSize = "12px";
                clearButton.style.padding = "6px 8px";

                actions.appendChild(selectAllButton);
                actions.appendChild(clearButton);
                panel.appendChild(actions);

                const distinctValues = getDistinctValues(column);
                const currentSelection = filterState[column]
                    ? new Set(filterState[column])
                    : new Set(distinctValues);

                if (distinctValues.length === 0) {{
                    const emptyInfo = document.createElement("div");
                    emptyInfo.className = "muted";
                    emptyInfo.textContent = "Keine Werte vorhanden";
                    panel.appendChild(emptyInfo);
                }} else {{
                    distinctValues.forEach((value) => {{
                        const option = document.createElement("label");
                        option.className = "filter-option";

                        const checkbox = document.createElement("input");
                        checkbox.type = "checkbox";
                        checkbox.value = value;
                        checkbox.checked = currentSelection.has(value);

                        checkbox.addEventListener("change", () => {{
                            const selected = new Set(
                                Array.from(panel.querySelectorAll('input[type="checkbox"]:checked')).map((node) => node.value)
                            );

                            if (selected.size === distinctValues.length) {{
                                delete filterState[column];
                            }} else {{
                                filterState[column] = selected;
                            }}

                            render();
                        }});

                        const text = document.createElement("span");
                        text.textContent = value === "" ? "(leer)" : value;

                        option.appendChild(checkbox);
                        option.appendChild(text);
                        panel.appendChild(option);
                    }});
                }}

                selectAllButton.addEventListener("click", () => {{
                    delete filterState[column];
                    render();
                }});

                clearButton.addEventListener("click", () => {{
                    filterState[column] = new Set();
                    render();
                }});

                wrapper.appendChild(headerMain);
                wrapper.appendChild(filterToggle);
                th.appendChild(wrapper);
                th.appendChild(panel);
                row.appendChild(th);
            }});

            thead.replaceChildren(row);
        }}

        function renderBody(rows) {{
            if (rows.length === 0) {{
                const tr = document.createElement("tr");
                const td = document.createElement("td");
                td.colSpan = columns.length;
                td.className = "empty-state";
                td.textContent = "Keine Daten vorhanden";
                tr.appendChild(td);
                tbody.replaceChildren(tr);
                return;
            }}

            const fragment = document.createDocumentFragment();

            rows.forEach((row) => {{
                const tr = document.createElement("tr");
                columns.forEach((column) => {{
                    const td = document.createElement("td");
                    td.innerHTML = escapeHtml(normalizeValue(row[column]));
                    tr.appendChild(td);
                }});
                fragment.appendChild(tr);
            }});

            tbody.replaceChildren(fragment);
        }}

        function render() {{
            renderHeader();
            const rows = getVisibleRows();
            renderBody(rows);
            updateStatus(rows);
        }}

        document.addEventListener("click", (event) => {{
            const target = event.target;
            if (!target.closest(".filter-panel") && !target.closest(".filter-toggle")) {{
                document.querySelectorAll(".filter-panel.open").forEach((node) => node.classList.remove("open"));
            }}
        }});

        resetSortButton.addEventListener("click", () => {{
            sortState.column = null;
            sortState.direction = null;
            render();
        }});

        resetFilterButton.addEventListener("click", () => {{
            Object.keys(filterState).forEach((key) => delete filterState[key]);
            render();
        }});

        render();
    </script>
</body>
</html>
"""


class MigManLoad:
    """
    Handles loading of transformed MigMan data into target system.
    """

    def __init__(
        self,
        nl: NemoLibrary,
        cfg: ConfigMigMan,
        logger: Union[logging.Logger, object],
        fh: ETLFileHandler,
        local_database: ETLDuckDBHandler,
    ) -> None:
        self.nl = nl
        self.cfg = cfg
        self.logger = logger
        self.fh = fh
        self.local_database = local_database
        self.migman_database = MigManUtils.MigManDatabaseLoad()
        super().__init__()

    def _nemo_project_name(self, project: str) -> str:
        return f"{self.cfg.load.nemo_project_prefix}{project}"

    def load(self) -> None:
        self.logger.info("Loading all MigMan objects")

        if self.cfg.load.delete_projects_before_load:
            self.logger.info("Deleting existing MigMan projects before load")
            project_ids = []
            for project in self.cfg.setup.projects:
                projectid = self.nl.getProjectID(self._nemo_project_name(project))
                if projectid:
                    project_ids.append(projectid)
            self.nl.deleteProjects(project_ids)

        # load objects - standard first
        for project in self.cfg.setup.projects:
            self.logger.info(f"Creating/updating project in NEMO: {project}")
            project, postfix = MigManUtils.split_migman_project_name(project)
            self._create_update_nemo_project(project=project, addon=None, postfix=postfix)

        for feature_assignment in self.cfg.setup.multi_projects_feature_assignments:
            self.logger.info(
                f"Creating/updating multi-project for feature assignments in NEMO: {feature_assignment}"
            )
            self._create_update_nemo_project(
                project="Features (Assignments)", addon=feature_assignment, postfix=""
            )

        for text in self.cfg.setup.multi_projects_texts:
            self.logger.info(f"Creating/updating multi-project for texts in NEMO: {text}")
            self._create_update_nemo_project(
                project="Features (Assignments)", addon=text, postfix=""
            )

    def _create_update_nemo_project(
        self, project: str, addon: str | None, postfix: str
    ) -> None:
        # do we have data to load?
        table_name = self.local_database.latest_table_name(
            steps=MigManTransformStep, maxstep=None, entity=project
        )
        if table_name is None:
            raise ValueError(f"No table found for entity {project}")

        # does project exist already?
        nemo_project_name = self._nemo_project_name(
            MigManUtils.MigManProjectName(project, addon, postfix)
        )
        projectid = self.nl.getProjectID(nemo_project_name)

        if (
            self.cfg.load.development_deficiency_mining_only
            or self.cfg.load.development_load_reports_only
        ) and not projectid:
            raise ValueError(
                f"Project {nemo_project_name} does not exist. Cannot proceed with development deficiency mining only mode."
            )

        # Get all column names
        columns_info = self.local_database.con.execute(
            f"SELECT name FROM pragma_table_info('{table_name}')"
        ).fetchall()

        if not columns_info:
            raise ValueError(f"Failed to retrieve columns for table {table_name}")

        columns = [col[0] for col in columns_info]

        # ------------------------------------------------------------
        # Option B: keep audit columns (Orig_*) in DuckDB,
        # but exclude them from:
        #   - MigMan model validation / column creation
        #   - upload to NEMO
        # ------------------------------------------------------------
        special_columns = {
            "duplicate_partners_json",
            "duplicate_top_score",
            "duplicate_match_count",
            "duplicate_group_key",
            "duplicate_group_values",
            "duplicate_recommended_id",
        }

        def skip_for_nemo(colname: str) -> bool:
            return colname.startswith("Orig_") or (colname in special_columns)

        columns_for_nemo = [c for c in columns if not skip_for_nemo(c)]

        # Build a temporary view for upload that contains only allowed columns
        upload_table_name = table_name
        if len(columns_for_nemo) != len(columns):
            tmp_view = f"__nemo_upload_{re.sub(r'[^0-9A-Za-z_]+', '_', table_name)}"
            select_cols = ", ".join([f'"{c}"' for c in columns_for_nemo])
            self.local_database.con.execute(f'DROP VIEW IF EXISTS "{tmp_view}"')
            self.local_database.con.execute(
                f'CREATE TEMP VIEW "{tmp_view}" AS SELECT {select_cols} FROM "{table_name}"'
            )
            self.logger.info(
                f"Upload view '{tmp_view}' created: filtered {len(columns) - len(columns_for_nemo)} columns (Orig_* / special)"
            )
            upload_table_name = tmp_view

        if projectid is None:
            self.logger.info(
                f"Project does not exist. Creating NEMO project: {nemo_project_name}"
            )
            self.nl.createProjects(
                projects=[
                    Project(
                        displayName=nemo_project_name,
                        description=f"Data Model for Mig Man table '{project}', addon '{addon}', postfix '{postfix}'",
                    )
                ],
            )

            new_columns: list[Column] = []
            for column_name in columns_for_nemo:
                # search column in migman database
                mig_man_col = next(
                    (
                        col
                        for col in self.migman_database
                        if col.project == project
                        and col.postfix == postfix
                        and col.nemo_import_name == column_name
                    ),
                    None,
                )
                if mig_man_col is None:
                    raise ValueError(
                        f"Column '{column_name}' not found in MigMan database for project '{project}' and postfix '{postfix}'"
                    )

                description = "\n".join(
                    f"{k}: {v if v else '<None>'}"
                    for k, v in mig_man_col.to_dict().items()
                )

                data_classification = DATA_CLASSIFICATIONS.get(
                    mig_man_col.nemo_import_name, None
                )
                if data_classification:
                    description += f"\n\nData Classification: {data_classification}"

                new_columns.append(
                    Column(
                        displayName=mig_man_col.nemo_display_name,
                        importName=mig_man_col.nemo_import_name,
                        internalName=mig_man_col.nemo_internal_name,
                        description=description,
                        dataType="string",
                        columnType="ExportedColumn",
                        dataClassificationInternalName=data_classification,
                        order=f"{mig_man_col.index:04d}",
                    )
                )

                self.logger.info(
                    f"Prepared column {mig_man_col.index:04d} '{column_name}' for NEMO project '{nemo_project_name}'"
                )

            self.logger.info(
                f"Creating {len(new_columns)} columns in NEMO project {nemo_project_name}"
            )
            self.nl.createColumns(
                projectname=nemo_project_name,
                columns=new_columns,
            )

        if (
            not self.cfg.load.development_deficiency_mining_only
            and not self.cfg.load.development_load_reports_only
        ):
            self.logger.info(f"Loading data into {nemo_project_name}")

            self.local_database.upload_table_to_nemo(
                table_name=upload_table_name,
                project_name=nemo_project_name,
                delete_temp_files=self.cfg.load.delete_temp_files,
            )

        if not self.cfg.load.development_load_reports_only:
            self._update_deficiency_mining(
                nemo_project_name=nemo_project_name,
                project_name=project,
                postfix=postfix,
                columns_in_file=columns_for_nemo,
            )

        if not self.cfg.load.development_deficiency_mining_only:
            self._load_reports(
                nemo_project_name=nemo_project_name,
                project_name=project,
                addon=addon,
                postfix=postfix,
            )

    def _update_deficiency_mining(
        self,
        nemo_project_name: str,
        project_name: str,
        postfix: str,
        columns_in_file: list[str],
    ) -> None:
        # create column specific fragments
        frags_checked: list[str] = []
        frags_msg: list[str] = []
        error_definitions: list[dict[str, str]] = []
        sorted_columns: list[str] = []
        joins: dict[str, dict[str, str]] = {}

        migman_fields = [
            x
            for x in self.migman_database
            if x.project == project_name
            and x.postfix == postfix
            and x.nemo_import_name in columns_in_file
        ]

        for migman_field in migman_fields:
            frag_check: list[str] = []
            frag_msg: list[str] = []

            # global checks
            if migman_field.snow_mandatory:
                frag_check.append(
                    f"{migman_field.nemo_internal_name} IS NULL OR {migman_field.nemo_internal_name} = ''"
                )
                frag_msg.append(
                    f"{migman_field.nemo_display_name} is mandatory and must not be empty"
                )

            # data type specific checks
            match migman_field.desc_section_data_type.lower():
                case "character":
                    match_len = re.search(r"x\((\d+)\)", migman_field.desc_section_format)
                    field_length = (
                        int(match_len.group(1))
                        if match_len
                        else len(migman_field.desc_section_format)
                    )
                    frag_check.append(
                        f"LENGTH({migman_field.nemo_internal_name}) > {field_length}"
                    )
                    frag_msg.append(
                        f"{migman_field.nemo_display_name} exceeds field length (max {field_length} digits)"
                    )

                case "integer" | "decimal":
                    format_str = migman_field.desc_section_format
                    value = migman_field.nemo_internal_name

                    has_leading_minus = format_str.startswith("-")
                    has_trailing_minus = format_str.endswith("-")
                    minus_allowed = has_leading_minus or has_trailing_minus

                    if not minus_allowed:
                        frag_check.append(f"INSTR({value}, '-') > 0")
                        frag_msg.append(
                            f"{migman_field.nemo_display_name} must not contain a minus sign (expected format: {format_str})"
                        )
                    else:
                        if has_leading_minus:
                            frag_check.append(
                                f"(INSTR({value}, '-') > 0 AND LEFT({value}, 1) != '-')"
                            )
                            frag_msg.append(
                                f"{migman_field.nemo_display_name} must have minus sign only at the beginning if present (expected format: {format_str})"
                            )
                        elif has_trailing_minus:
                            frag_check.append(
                                f"(INSTR({value}, '-') > 0 AND RIGHT({value}, 1) != '-')"
                            )
                            frag_msg.append(
                                f"{migman_field.nemo_display_name} must have minus sign only at the end if present (expected format: {format_str})"
                            )

                    value_stripped = f"REPLACE({value}, '-', '')"

                    if "." in format_str:
                        decimals_allowed = sum(
                            1
                            for c in format_str.split(".")[1].lower()
                            if c in ("z", "9")
                        )
                    else:
                        decimals_allowed = 0

                    if decimals_allowed == 0:
                        frag_check.append(f"INSTR({value_stripped}, ',') > 0")
                        frag_msg.append(
                            f"{migman_field.nemo_display_name} must not contain decimal places (expected format: {format_str})"
                        )
                    else:
                        frag_check.append(
                            f"""LOCATE(',', {value_stripped}) > 0 AND 
                                LENGTH(RIGHT(
                                    {value_stripped},
                                    LENGTH({value_stripped}) - LOCATE(',', {value_stripped})
                                )) > {decimals_allowed}"""
                        )
                        frag_msg.append(
                            f"{migman_field.nemo_display_name} has too many decimal places (maximum {decimals_allowed} allowed; expected format: {format_str})"
                        )

                    format_clean = format_str.replace("-", "")
                    integer_format_part = format_clean.split(".")[0]
                    digits_before_comma = sum(
                        1 for c in integer_format_part.lower() if c in ("z", "9")
                    )

                    cleaned = f"REPLACE(REPLACE(REPLACE({value_stripped}, ' ', ''), '.', ''), ',', '')"

                    frag_check.append(
                        f"""LENGTH(
                            CASE 
                                WHEN LOCATE(',', {value_stripped}) > 0 
                                THEN LEFT({cleaned}, LOCATE(',', {value_stripped}) - 1)
                                ELSE {cleaned}
                            END
                        ) > {digits_before_comma}"""
                    )
                    frag_msg.append(
                        f"{migman_field.nemo_display_name} has too many digits before the decimal point (maximum {digits_before_comma} allowed; expected format: {format_str})"
                    )

                    frag_check.append(
                        f"""NOT REPLACE({value}, '-', '') 
                        LIKE_REGEXPR('^[-]?([[:digit:]]+|[[:digit:]]{{1,3}}(\\.[[:digit:]]{{3}})*)(,[[:digit:]]+)?$')"""
                    )
                    frag_msg.append(
                        f"{migman_field.nemo_display_name} is not a valid number (expected German format, e.g. 1.234,56; format: {format_str})"
                    )

                case "date":
                    pattern = "^(0[1-9]|[1-2][0-9]|3[0-1])\\.(0[1-9]|1[0-2])\\.([0-9]{4})$"
                    frag_check.append(
                        f"NOT {migman_field.nemo_internal_name} LIKE_REGEXPR('{pattern}')"
                    )
                    frag_msg.append(
                        f"{migman_field.nemo_display_name} is not a valid date"
                    )

            # special fields
            if "mail" in migman_field.nemo_internal_name:
                frag_check.append(
                    f"NOT {migman_field.nemo_internal_name} LIKE_REGEXPR '^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$'"
                )
                frag_msg.append(
                    f"{migman_field.nemo_display_name} does not match the general email format"
                )

            # VAT_ID
            if "s_ustid_ustid" in migman_field.nemo_internal_name:
                joins[migman_field.nemo_internal_name] = {"CLASSIFICATION": "VAT_ID"}
                frag_check.append(
                    f"(genius_{migman_field.nemo_internal_name}.STATUS IS NOT NULL AND genius_{migman_field.nemo_internal_name}.STATUS = 'check')"
                )
                frag_msg.append(
                    f"genius analysis: ' || genius_{migman_field.nemo_internal_name}.STATUS_MESSAGE || '"
                )

            # URL
            elif "s_adresse_homepage" in migman_field.nemo_internal_name:
                joins[migman_field.nemo_internal_name] = {"CLASSIFICATION": "URL"}
                frag_check.append(
                    f"(genius_{migman_field.nemo_internal_name}.STATUS IS NOT NULL AND genius_{migman_field.nemo_internal_name}.STATUS = 'check')"
                )
                frag_msg.append(
                    f"genius analysis: ' || genius_{migman_field.nemo_internal_name}.STATUS_MESSAGE || '"
                )

            # now build deficiency mining report for this column (if there are checks)
            if frag_check:
                for check, msg in zip(frag_check, frag_msg):
                    error_no = f"{len(error_definitions) + 1:03d}"
                    flag_alias = _beautify_report_column_name(msg)
                    count_alias = _beautify_report_column_name(f"{msg} Count")
                    error_definitions.append(
                        {
                            "error_no": error_no,
                            "field_internal_name": migman_field.nemo_internal_name,
                            "field_display_name": migman_field.nemo_display_name,
                            "message": msg,
                            "flag_alias": flag_alias,
                            "count_alias": count_alias,
                            "quoted_flag_alias": _quote_sql_identifier(flag_alias),
                            "quoted_count_alias": _quote_sql_identifier(count_alias),
                            "check": check,
                        }
                    )

                frags_checked.extend(frag_check)
                frags_msg.extend(frag_msg)

                sorted_columns = [
                    f'{migman_field.nemo_internal_name} AS "{migman_field.nemo_display_name}"'
                ] + [
                    f'{other_field.nemo_internal_name} AS "{other_field.nemo_display_name}"'
                    for other_field in self.migman_database
                    if other_field.project == project_name
                    and other_field.postfix == postfix
                    and other_field.index != migman_field.index
                    and other_field.nemo_import_name in columns_in_file
                ]

                case_statement_specific = " ||\n\t".join(
                    [
                        f"CASE\n\t\tWHEN {check}\n\t\tTHEN CHAR(10) || '{msg}'\n\t\tELSE ''\n\tEND"
                        for check, msg in zip(frag_check, frag_msg)
                    ]
                )

                status_conditions = " OR ".join(frag_check)

                sql_statement = f"""SELECT
\tCASE 
\t\tWHEN {status_conditions} THEN 'check'
\tELSE 'ok'
\tEND AS STATUS
\t,LTRIM({case_statement_specific},CHAR(10)) AS DEFICIENCY_MINING_MESSAGE
\t,{',\n\t'.join(sorted_columns)}
FROM
\t$schema.$table
"""
                if migman_field.nemo_internal_name in joins:
                    sql_statement += f"""LEFT JOIN
\t$schema.SHARED_NAIGENT genius_{migman_field.nemo_internal_name}
ON  
\t    genius_{migman_field.nemo_internal_name}.CLASSIFICATION = '{joins[migman_field.nemo_internal_name]["CLASSIFICATION"]}'
\tAND genius_{migman_field.nemo_internal_name}.VALUE          = {migman_field.nemo_internal_name}
"""

                report_display_name = (
                    f"(DEFICIENCIES) {migman_field.index:03} {migman_field.nemo_display_name}"
                )
                report_internal_name = get_internal_name(report_display_name)

                self.nl.createReports(
                    projectname=nemo_project_name,
                    reports=[
                        Report(
                            displayName=report_display_name,
                            internalName=report_internal_name,
                            querySyntax=sql_statement,
                            description=f"Deficiency Mining Report for column '{migman_field.nemo_display_name}' in project '{project_name}'",
                        )
                    ],
                )

                self.nl.createRules(
                    projectname=nemo_project_name,
                    rules=[
                        Rule(
                            displayName=f"DM_{migman_field.index:03}: {migman_field.nemo_display_name}",
                            ruleSourceInternalName=report_internal_name,
                            ruleGroup="02 Columns",
                            description=f"Deficiency Mining Rule for column '{migman_field.nemo_display_name}' in project '{project_name}'",
                        )
                    ],
                )

                self.logger.info(
                    f"project: {project_name}, column: {migman_field.nemo_display_name}: {len(frag_check)} frags added"
                )

        case_statement_specific, status_conditions = self._create_dm_rule_global(
            nemo_project_name=nemo_project_name,
            project_name=project_name,
            postfix=postfix,
            columns_in_file=columns_in_file,
            frags_checked=frags_checked,
            frags_msg=frags_msg,
            sorted_columns=sorted_columns,
            joins=joins,
        )

        self._create_report_for_migman(
            nemo_project_name=nemo_project_name,
            project_name=project_name,
            postfix=postfix,
            columns_in_file=columns_in_file,
            case_statement_specific=case_statement_specific,
            status_conditions=status_conditions,
            joins=joins,
        )

        self._create_report_for_customer(
            nemo_project_name=nemo_project_name,
            project_name=project_name,
            postfix=postfix,
            columns_in_file=columns_in_file,
            case_statement_specific=case_statement_specific,
            status_conditions=status_conditions,
            joins=joins,
            error_definitions=error_definitions,
        )

        self.logger.info(f"Project {project_name}: {len(frags_checked)} checks implemented...")

    def _create_dm_rule_global(
        self,
        nemo_project_name: str,
        project_name: str,
        postfix: str,
        columns_in_file: list[str],
        frags_checked: list[str],
        frags_msg: list[str],
        sorted_columns: list[str],
        joins: dict[str, dict[str, str]],
    ) -> tuple[str, str]:
        case_statement_specific = " ||\n\t".join(
            [
                f"CASE\n\t\tWHEN {check}\n\t\tTHEN  CHAR(10) || '{msg}'\n\t\tELSE ''\n\tEND"
                for check, msg in zip(frags_checked, frags_msg)
            ]
        )

        status_conditions = " OR ".join(frags_checked)

        sql_statement = f"""WITH CTEDefMining AS (
SELECT
\t\t{',\n\t\t'.join([x.nemo_internal_name for x in self.migman_database if x.project == project_name and x.postfix == postfix and x.nemo_import_name in columns_in_file])}
    ,LTRIM({case_statement_specific},CHAR(10)) AS DEFICIENCY_MINING_MESSAGE
    ,CASE 
        WHEN {status_conditions} THEN 'check'
        ELSE 'ok'
    END AS STATUS
FROM
    $schema.$table"""

        for join in joins:
            sql_statement += f"""
LEFT JOIN
\t$schema.SHARED_NAIGENT genius_{join}
ON  
\t    genius_{join}.CLASSIFICATION = '{joins[join]["CLASSIFICATION"]}'
\tAND genius_{join}.VALUE          = {join}"""

        sql_statement += """
)
SELECT
    Status
    , DEFICIENCY_MINING_MESSAGE
    , {cols}
FROM 
    CTEDefMining""".format(cols=",\n\t".join(sorted_columns))

        report_display_name = "(DEFICIENCIES) GLOBAL"
        report_internal_name = get_internal_name(report_display_name)

        self.nl.createReports(
            projectname=nemo_project_name,
            reports=[
                Report(
                    displayName=report_display_name,
                    internalName=report_internal_name,
                    querySyntax=sql_statement,
                    description=f"Deficiency Mining Report for  project '{project_name}'",
                )
            ],
        )

        self.nl.createRules(
            projectname=nemo_project_name,
            rules=[
                Rule(
                    displayName="Global",
                    ruleSourceInternalName=report_internal_name,
                    ruleGroup="01 Global",
                    description=f"Deficiency Mining Rule for project '{project_name}'",
                )
            ],
        )

        self.logger.info(
            f"nemo project: {nemo_project_name}, global rule created with {len(frags_checked)} checks"
        )
        return case_statement_specific, status_conditions

    def _create_report_for_migman(
        self,
        nemo_project_name: str,
        project_name: str,
        postfix: str,
        columns_in_file: list[str],
        case_statement_specific: str,
        status_conditions: str,
        joins: dict[str, dict[str, str]],
    ) -> None:
        sql_statement = f"""WITH CTEDefMining AS (
    SELECT
        {',\n\t\t'.join([x.nemo_internal_name for x in self.migman_database if x.project == project_name and x.postfix == postfix and x.nemo_import_name in columns_in_file])}
        ,LTRIM({case_statement_specific},CHAR(10)) AS DEFICIENCY_MINING_MESSAGE
        ,CASE 
            WHEN {status_conditions} THEN 'check'
            ELSE 'ok'
        END AS STATUS
    FROM
        $schema.$table"""

        for join in joins:
            sql_statement += f"""
LEFT JOIN
\t$schema.SHARED_NAIGENT genius_{join}
ON  
\t    genius_{join}.CLASSIFICATION = '{joins[join]["CLASSIFICATION"]}'
\tAND genius_{join}.VALUE          = {join}"""

        sql_statement += """
)
SELECT
    {cols}
FROM 
    CTEDefMining
WHERE
    STATUS = 'ok'
    """.format(
            cols=",\n\t".join(
                [
                    f'{x.nemo_internal_name} as "{x.header_section_label}"'
                    for x in self.migman_database
                    if x.project == project_name
                    and x.postfix == postfix
                    and x.nemo_import_name in columns_in_file
                ]
            )
        )

        report_display_name = "(MigMan) All records with no message"
        report_internal_name = get_internal_name(report_display_name)

        self.nl.createReports(
            projectname=nemo_project_name,
            reports=[
                Report(
                    displayName=report_display_name,
                    internalName=report_internal_name,
                    querySyntax=sql_statement,
                    description=f"MigMan export with valid data for project '{project_name}'",
                )
            ],
        )

        self.logger.info(f"Created MigMan report for project '{nemo_project_name}'")

    def _create_report_for_customer(
        self,
        nemo_project_name: str,
        project_name: str,
        postfix: str,
        columns_in_file: list[str],
        case_statement_specific: str,
        status_conditions: str,
        joins: dict[str, dict[str, str]],
        error_definitions: list[dict[str, str]],
    ) -> None:
        base_columns = [
            x.nemo_internal_name
            for x in self.migman_database
            if x.project == project_name
            and x.postfix == postfix
            and x.nemo_import_name in columns_in_file
        ]

        flag_columns = [
            f"CASE WHEN {err['check']} THEN 1 ELSE 0 END AS {err['quoted_flag_alias']}"
            for err in error_definitions
        ]

        count_columns = [
            f"SUM({err['quoted_flag_alias']}) OVER () AS {err['quoted_count_alias']}"
            for err in error_definitions
        ]

        error_code_concat = " || ".join(
            [f"TO_NVARCHAR({err['quoted_flag_alias']})" for err in error_definitions]
        )
        if not error_code_concat:
            error_code_concat = "''"

        record_pipe = " || '|' || ".join(
            [f"COALESCE(TO_NVARCHAR({col}), '')" for col in base_columns]
        )
        if not record_pipe:
            record_pipe = "''"

        final_select_columns: list[str] = []
        for err in error_definitions:
            final_select_columns.append(err["quoted_flag_alias"])
            final_select_columns.append(err["quoted_count_alias"])

        final_select_columns.extend(
            [
                "ERROR_CODE_CONCAT",
                "ERROR_CODE_COUNT_OVERALL",
                "RECORD_PIPE",
            ]
        )

        cte_def_mining_columns = base_columns + flag_columns + [
            "CASE\n"
            f"        WHEN {status_conditions} THEN 'check'\n"
            "        ELSE 'ok'\n"
            "    END AS STATUS"
        ]

        cte_def_mining_prepared_columns = ["CTEDefMining.*"] + count_columns + [
            f"{error_code_concat} AS ERROR_CODE_CONCAT",
            f"{record_pipe} AS RECORD_PIPE",
        ]

        sql_statement = """WITH CTEDefMining AS (
SELECT
    {def_mining_columns}
FROM
    $schema.$table""".format(
            def_mining_columns=",\n    ".join(cte_def_mining_columns)
        )

        for join in joins:
            sql_statement += f"""
LEFT JOIN
\t$schema.SHARED_NAIGENT genius_{join}
ON  
\t    genius_{join}.CLASSIFICATION = '{joins[join]["CLASSIFICATION"]}'
\tAND genius_{join}.VALUE          = {join}"""

        sql_statement += """
),
CTEDefMiningPrepared AS (
SELECT
    {prepared_columns}
FROM
    CTEDefMining
),
CTEDefMiningFinal AS (
SELECT
    CTEDefMiningPrepared.*,
    COUNT(*) OVER (PARTITION BY ERROR_CODE_CONCAT) AS ERROR_CODE_COUNT_OVERALL
FROM
    CTEDefMiningPrepared
)
SELECT
    {final_columns}
FROM
    CTEDefMiningFinal
WHERE
    STATUS <> 'ok'
""".format(
            prepared_columns=",\n    ".join(cte_def_mining_prepared_columns),
            final_columns=",\n    ".join(final_select_columns),
        )

        report_display_name = "(Customer) All records with message"
        report_internal_name = get_internal_name(report_display_name)

        self.nl.createReports(
            projectname=nemo_project_name,
            reports=[
                Report(
                    displayName=report_display_name,
                    internalName=report_internal_name,
                    querySyntax=sql_statement,
                    description=f"export invalid data for project '{project_name}'",
                )
            ],
        )
        self.logger.info(f"Created Customer report for project '{nemo_project_name}'")

    def _load_reports(
        self,
        nemo_project_name: str,
        project_name: str,
        addon: str,
        postfix: str,
    ) -> None:
        self.logger.info(f"Loading reports from {nemo_project_name}")

        data = [
            ("to_customer", "_with_messages", "(Customer) All records with message"),
            ("to_proalpha", "", "(MigMan) All records with no message"),
        ]

        report_folder = Path(self.cfg.etl_directory) / "reports"
        for folder, file_postfix, report_name in data:
            logging.info(
                f"Exporting '{project_name}', addon '{addon}', postfix '{postfix}', report name: '{report_name}' to '{folder}'"
            )
            file_name = report_folder / folder / f"{project_name}{file_postfix}.csv"
            file_name.parent.mkdir(parents=True, exist_ok=True)
            df = self.nl.LoadReport(
                projectname=nemo_project_name,
                report_name=report_name,
                data_types=str,
            )
            is_customer_report = report_name == "(Customer) All records with message"
            if is_customer_report:
                original_column_count = len(df.columns)
                df = _filter_customer_report_columns(df)
                self.logger.info(
                    "Filtered customer report columns with zero count: "
                    f"{original_column_count} -> {len(df.columns)}"
                )
            df.to_csv(
                file_name,
                index=False,
                sep=";",
                encoding="utf-8-sig",
            )
            html_file_name = file_name.with_suffix(".html")
            html_content = _create_deficiency_report_html(
                report_title=project_name,
                file_path=_relative_report_path(html_file_name),
                created_at=datetime.now(),
                df=df,
                error_row_count=len(df.index) if is_customer_report else 0,
                generated_by_version=f"MigMan / Nemo ETL Version {_get_nemo_etl_version()}",
            )
            html_file_name.write_text(html_content, encoding="utf-8")

            logging.info(
                f"File '{file_name}' for '{project_name}', addon '{addon}', postfix '{postfix}' exported '{report_name}'"
            )
