# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/scilint.ipynb.

# %% auto 0
__all__ = ['logger', 'tidy', 'get_default_spec', 'lint_nb', 'lint_nbs', 'display_warning_report', 'lint', 'build', 'scilint_tidy',
           'scilint_lint', 'scilint_build', 'scilint_ci']

# %% ../nbs/scilint.ipynb 2
import json
import logging
import operator
import os
import shutil
import sys
from importlib import reload
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, Tuple

import numpy as np
import pandas as pd
import yaml
from execnb.nbio import read_nb
from fastcore.script import Param, call_parse, store_false
from fastcore.xtras import globtastic
from nbdev.clean import nbdev_clean
from nbdev.doclinks import nbdev_export, nbglob
from nbdev.quarto import nbdev_docs, nbdev_readme
from nbdev.test import nbdev_test
from nbqa.find_root import find_project_root

from .indicators import indicator_funcs
from scilint.utils import (
    configure_logging,
    find_common_root,
    get_excluded_paths,
    get_project_root,
    is_nbdev_project,
    resolve_nbs,
    run_nbqa_cmd,
)

reload(logging)
logger = logging.getLogger(__name__)

# %% ../nbs/scilint.ipynb 8
def tidy(root_dir: Path = None):
    tidy_tools = ["black", "isort", "autoflake"]
    for cmd in tidy_tools:
        logger.debug(f"Running command {cmd} via nbQA CLI at root dir: {root_dir}")
        run_nbqa_cmd(cmd, root_dir)

# %% ../nbs/scilint.ipynb 12
def get_default_spec():
    return {
        "exclusions": None,
        "fail_over": 1,
        "out_dir": "/tmp/scilint/",
        "precision": 3,
        "print_syntax_errors": False,
        "evaluate": True,
        "nb_path_display": "parent",
        "warnings": {
            "lt": {
                "calls_per_func_median": 0,
                "calls_per_func_mean": 1,
                "in_func_pct": 20,
                "tests_func_coverage_pct": 20,
                "tests_per_func_mean": 0.5,
                "markdown_code_pct": 5,
            },
            "gt": {"total_code_len": 50000, "loc_per_md_section": 2000},
            "equals": {"has_syntax_error": True},
        },
    }

# %% ../nbs/scilint.ipynb 15
def lint_nb(
    spec_name: str,
    nb_path: Path,
    conf: Dict[str, Any],
    indicators: Dict[str, Callable],
    include_in_scoring: bool,
) -> Tuple[float]:
    nb = read_nb(nb_path)

    has_syntax_error = False
    indic_vals = list(np.repeat(np.nan, len(indicators)))
    try:
        for i, indic_name in enumerate(indicators):
            indic_vals[i] = round(indicators[indic_name](nb), conf["precision"])
    except SyntaxError as se:
        if conf["print_syntax_errors"]:
            logger.warn(f"Syntax error in notebook: {nb_path} reason: ", se)
        has_syntax_error = True
    indic_vals.append(has_syntax_error)
    indic_vals.append(include_in_scoring)
    indic_vals.insert(0, spec_name)

    return tuple(indic_vals)

# %% ../nbs/scilint.ipynb 18
def _calculate_warnings(
    spec_name: str,
    scoring_report: pd.DataFrame,
    conf: Dict[str, Any],
    include_missing: bool = False,
) -> Tuple[Dict[str, Any], int]:
    warning_details = []

    for op_text in list(conf["warnings"].keys()):
        for indic in conf["warnings"][op_text]:
            metric_series = scoring_report[indic]
            or_exp = pd.isnull(metric_series) if include_missing else False
            op = (
                operator.lt
                if op_text == "lt"
                else operator.gt
                if op_text == "gt"
                else operator.eq
            )
            warning_data = metric_series[
                (op(metric_series, conf["warnings"][op_text][indic])) | (or_exp)
            ]
            warning_dict = warning_data.to_dict()
            for key, val in warning_dict.items():
                warning_dict[key] = (
                    indic,
                    val,
                    op_text,
                    conf["warnings"][op_text][indic],
                )
            warning_details.append(warning_dict)

    all_warns = _reshape_warnings(spec_name, scoring_report, warning_details)
    num_warnings = len(all_warns)
    return all_warns, num_warnings

# %% ../nbs/scilint.ipynb 21
def _reshape_warnings(
    spec_name: str, scoring_report: pd.DataFrame, warning_details: Iterable[Any]
) -> Dict[str, Iterable[Tuple]]:
    warnings_by_nb = {nb: [] for nb in scoring_report.index}
    for nb in scoring_report.index:
        for wd in warning_details:
            if nb in wd:
                warnings_by_nb[nb].append(tuple([spec_name, nb] + list(wd[nb])))
    warnings_by_nb = {key: val for key, val in warnings_by_nb.items() if len(val) > 0}
    flattened_warns = [item for sublist in warnings_by_nb.values() for item in sublist]
    return pd.DataFrame.from_records(
        data=flattened_warns,
        columns=[
            "spec_name",
            "notebook",
            "indicator",
            "value",
            "operator",
            "threshold",
        ],
    )

# %% ../nbs/scilint.ipynb 23
def _get_nb_display_name(nb_path_display: str, nb_path: Path) -> str:
    nb_name = None
    if nb_path_display == "abs":
        nb_name = str(nb_path)
    elif nb_path_display == "parent":
        nb_name = str(Path(nb_path.parent.stem, nb_path.stem))
    elif nb_path_display == "stem":
        nb_name = nb_path.stem
    else:
        raise ValueError("nb_path_display has to be one of: \{stem, parent, abs\}")
    return nb_name

# %% ../nbs/scilint.ipynb 26
def lint_nbs(
    spec_name: str,
    conf: Dict[str, Any],
    indicators: Dict[str, Callable],
    nb_paths: Iterable[Path] = None,
    nb_glob: Path = None,
):
    if nb_paths is None:
        nb_paths = [Path(p).absolute() for p in nbglob(nb_glob)]
    else:
        nb_paths = [Path(p).absolute() for p in nb_paths]

    if len(nb_paths) == 0:
        raise ValueError("Attempt to lint notebooks but no files found")

    excluded_paths = None
    exclusions = conf["exclusions"]
    if exclusions is not None:
        excluded_paths = get_excluded_paths(nb_paths, exclude_pattern=exclusions)

    results = []
    nb_names = []
    for nb_path in nb_paths:
        include_in_scoring = True
        if exclusions is not None:
            include_in_scoring = False if nb_path in excluded_paths else True

        nb_names.append(_get_nb_display_name(conf["nb_path_display"], nb_path))

        lint_result = lint_nb(spec_name, nb_path, conf, indicators, include_in_scoring)
        results.append(lint_result)

    lint_report = pd.DataFrame.from_records(
        data=results,
        index=nb_names,
        columns=["spec_name"]
        + list(indicators.keys())
        + ["has_syntax_error", "include_in_scoring"],
    ).sort_values(["in_func_pct", "markdown_code_pct"], ascending=False)

    scoring_report = lint_report[lint_report.include_in_scoring].copy()
    all_warns, num_warnings = _calculate_warnings(spec_name, scoring_report, conf)
    return lint_report, all_warns, num_warnings

# %% ../nbs/scilint.ipynb 28
def _map_paths_specs(nb_glob: Path = None, specs_glob: Path = Path(".").resolve()):
    logger.debug(f"Mapping notebooks (glob={nb_glob}) to specs (glob={specs_glob})")
    nbs = resolve_nbs(nb_glob)
    if len(nbs) == 0:
        raise ValueError(f"Path glob expression: {nb_glob} - matched no notebooks")
    else:
        logger.debug(f"Path glob matched {len(nbs)} notebooks: {nbs}")
    spec_files = [
        Path(p).resolve()
        for p in globtastic(
            specs_glob,
            file_glob="scilint-*.yaml",
            skip_folder_re="ipynb_checkpoints|_proc",
        )
    ]
    logger.debug(f"Spec files: {spec_files}")
    default_spec_files = [p for p in spec_files if p.name == "scilint-default.yaml"]
    default_spec_file = default_spec_files[0] if len(default_spec_files) > 0 else None
    spec_dirs = [p.parent for p in spec_files]

    spec_nbs = {k: [] for k in spec_files}
    for nb in [Path(p) for p in nbs]:
        found_spec = False
        for name, spec_dir in zip(spec_files, spec_dirs):
            if nb.parent == spec_dir:
                spec_nbs[name].append(nb)
                found_spec = True
        if not found_spec:
            if default_spec_file is not None:
                spec_nbs[default_spec_file].append(nb)
            else:
                # Special case: not actually a valid file path - triggers loading a fallback
                logger.debug(f"No spec file found notebook: {nb} - using fallback spec")
                fallback_path = Path("scilint-default")
                if fallback_path not in spec_nbs:
                    spec_nbs[fallback_path] = []
                spec_nbs[fallback_path].append(nb)

    logger.debug(f"Specs to notebooks map:\n{spec_nbs}")
    return spec_nbs

# %% ../nbs/scilint.ipynb 44
def display_warning_report(all_warns: pd.DataFrame):
    print(
        "\n******************************************Begin Scilint Warning Report*****************************************"
    )
    print(all_warns.to_markdown(tablefmt="grid", index=False))
    print(
        "\n******************************************End Scilint Warning Report*******************************************\n"
    )

# %% ../nbs/scilint.ipynb 46
def _persist_results(
    lint_report: pd.DataFrame, all_warns: pd.DataFrame, conf: Dict[str, Any]
):
    out_dir = Path(conf["out_dir"])
    conf_to_persist = {k: v for k, v in conf.items() if k != "indicators"}
    if not out_dir.exists():
        Path(out_dir).mkdir()
    with open(Path(out_dir, "scilint_config.json"), "w") as outfile:
        json.dump(conf_to_persist, outfile)
    all_warns.to_csv(Path(out_dir, "scilint_warnings.csv"), index=False)
    lint_report.to_csv(Path(out_dir, "scilint_report.csv"))

# %% ../nbs/scilint.ipynb 50
def _load_conf(
    conf_path: str = None,
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    if conf_path is None:
        project_root = find_project_root(tuple([str(Path(".").resolve())]))
        conf_path = Path(project_root, "nbs", "scilint-default.yaml")
        logger.info(f"Loading default lint config: {conf_path}")
    else:
        conf_path = Path(conf_path)

    logger.debug(f"Loading config from: {conf_path}")
    conf = yaml.safe_load(conf_path.read_text())
    logger.debug(f"Loaded configuration\n {conf}")
    conf["nb_path_display"]
    override_names = (
        "exclusions",
        "fail_over",
        "out_dir",
        "precision",
        "print_syntax_errors",
    )
    overrides = (exclusions, fail_over, out_dir, precision, print_syntax_errors)
    for override in zip(override_names, overrides):
        if override[1] is not None:
            conf[override[0]] = override[1]
    return conf

# %% ../nbs/scilint.ipynb 54
def lint(
    display_report: bool = True,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
    exit_on_failure: bool = True,
):
    exit_code = 0
    spec_nbs = _map_paths_specs(nb_glob, specs_glob)
    lint_reports = []
    all_warns = []
    warns_count = []
    linting_failure = False
    for spec, nbs in spec_nbs.items():
        if len(nbs) == 0:
            print(
                f"Linting skipped for: {spec.name} as no notebooks found matching path expression"
            )
            continue
        if str(spec) == "scilint-default":
            logger.info("No spec assignment found - using fallback spec configuration")
            conf = get_default_spec()
        else:
            conf = _load_conf(
                spec, exclusions, fail_over, out_dir, precision, print_syntax_errors
            )
            conf["nb_path_display"]
        if conf["evaluate"] == False:
            print(f"Linting skipped for: {spec.name} as evaluate is set to false")
            continue
        lint_report, report_warns, num_warnings = lint_nbs(
            spec.name, conf, indicator_funcs, nb_paths=nbs
        )
        lint_reports.append(lint_report)
        all_warns.append(report_warns)
        warns_count.append(num_warnings)

        fail_over_conf = conf["fail_over"]
        if conf["fail_over"] == -1:
            print(f"Linting warnings ignored for: {spec.name} as fail_over set to -1")
        elif num_warnings == 0:
            print(f"Linting success for: {spec.name}, no issues found")
        elif num_warnings <= conf["fail_over"]:
            print(
                f"Linting success for: {spec.name}, warnings ({num_warnings}) <= than threshold ({fail_over_conf}) "
            )
        else:
            print(
                f"Linting failed for: {spec.name}, total warnings ({num_warnings}) exceeded threshold ({fail_over_conf})"
            )
            linting_failure = True
            continue

    lint_report = pd.concat(lint_reports) if len(lint_reports) > 0 else lint_report
    all_warns = pd.concat(all_warns) if len(all_warns) > 0 else report_warns
    num_warnings = sum(warns_count)

    _persist_results(lint_report, all_warns, conf)

    if num_warnings > 0:
        if display_report:
            display_warning_report(all_warns)
        if not linting_failure:
            print(
                f"{num_warnings} warnings founds, within tolerated thresholds for all specs"
            )
        elif exit_on_failure:
            sys.exit(num_warnings)
        else:
            exit_code = num_warnings
    elif num_warnings == 0:
        print("No issues found during linting")
    return exit_code

# %% ../nbs/scilint.ipynb 61
def build(
    display_report: bool = True,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
    exit_on_failure: bool = True,
):
    print("Tidying notebooks..")
    tidy(nb_glob)
    if is_nbdev_project():
        nbdev_export.__wrapped__()
        print("Converted notebooks to modules")
        print("Testing notebooks..")
        nbdev_test.__wrapped__()
    print("Running notebook linter..")
    lint(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
        exit_on_failure,
    )
    if is_nbdev_project():
        nbdev_clean.__wrapped__()
        print("Cleaned notebooks")

# %% ../nbs/scilint.ipynb 64
@call_parse
def scilint_tidy(nb_glob: Path = None, log_level: str = "warn"):
    configure_logging(log_level)
    common_root = find_common_root(nb_glob)
    logger.debug(f"Tidying from: {common_root}")
    tidy(common_root)

# %% ../nbs/scilint.ipynb 66
@call_parse
def scilint_lint(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: str = None,
    precision: int = None,
    print_syntax_errors: bool = None,
    exit_on_failure: bool = True,
    log_level: str = "warn",
):
    configure_logging(log_level)
    lint(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

# %% ../nbs/scilint.ipynb 69
@call_parse
def scilint_build(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
    exit_on_failure: bool = True,
    log_level: str = "warn",
):
    configure_logging(log_level)
    build(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

# %% ../nbs/scilint.ipynb 71
@call_parse
def scilint_ci(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
    exit_on_failure: bool = True,
    log_level: str = "warn",
):
    configure_logging(log_level)
    if not is_nbdev_project():
        print("scilint_ci feature is only available for nbdev projects")
        return

    build(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

    if not shutil.which("quarto"):
        print(
            "Quarto is not installed. A working quarto install is required for the CI build"
        )
        sys.exit(-1)
    nbdev_readme.__wrapped__()
    nbdev_docs.__wrapped__()
