# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/scilint.ipynb.

# %% auto 0
__all__ = ['run_nbqa_cmd', 'tidy', 'get_project_root', 'is_nbdev_project', 'get_default_spec', 'lint_nb', 'lint_nbs',
           'display_warning_report', 'lint', 'build', 'scilint_tidy', 'scilint_lint', 'scilint_build', 'scilint_ci']

# %% ../nbs/scilint.ipynb 2
import json
import operator
import os
import shutil
import sys
from configparser import InterpolationMissingOptionError
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, Tuple

import numpy as np
import pandas as pd
import yaml
from execnb.nbio import read_nb
from fastcore.script import Param, call_parse, store_false
from fastcore.xtras import globtastic
from nbdev.clean import nbdev_clean
from nbdev.config import get_config
from nbdev.doclinks import nbdev_export, nbglob
from nbdev.quarto import nbdev_docs, nbdev_readme
from nbdev.test import nbdev_test
from nbqa.__main__ import _get_configs, _main
from nbqa.cmdline import CLIArgs
from nbqa.find_root import find_project_root

from .indicators import indicator_funcs

# %% ../nbs/scilint.ipynb 7
def run_nbqa_cmd(cmd):
    print(f"Running {cmd}")
    project_root: Path = find_project_root(tuple([str(Path(".").resolve())]))
    args = CLIArgs.parse_args([cmd, str(project_root)])
    configs = _get_configs(args, project_root)
    output_code = _main(args, configs)
    return output_code

# %% ../nbs/scilint.ipynb 9
def tidy():
    tidy_tools = ["black", "isort", "autoflake"]
    [run_nbqa_cmd(c) for c in tidy_tools]

# %% ../nbs/scilint.ipynb 11
def get_project_root(path: Path = Path(".").resolve()):
    return find_project_root(tuple([str()]))

# %% ../nbs/scilint.ipynb 12
def is_nbdev_project(project_path: Path = Path(".")):
    is_nbdev = True
    project_root = find_project_root(tuple([str(project_path.resolve())]))

    if not Path(project_root, "settings.ini").exists():
        is_nbdev = False
    try:
        get_config().lib_name
    except InterpolationMissingOptionError:
        is_nbdev = False

    return is_nbdev

# %% ../nbs/scilint.ipynb 15
def get_default_spec():
    return {
        "exclusions": None,
        "fail_over": 1,
        "out_dir": "/tmp/scilint/",
        "precision": 3,
        "print_syntax_errors": False,
        "evaluate": True,
        "warnings": {
            "lt": {
                "calls_per_func_median": 0,
                "calls_per_func_mean": 1,
                "in_func_pct": 20,
                "asserts_func_ratio": 1,
                "tests_func_coverage_pct": 20,
                "tests_per_func_mean": 0.5,
                "markdown_code_pct": 5,
            },
            "gt": {"total_code_len": 50000, "loc_per_md_section": 2000},
            "equals": {"has_syntax_error": True},
        },
    }

# %% ../nbs/scilint.ipynb 18
def lint_nb(
    spec_name: str,
    nb_path: Path,
    conf: Dict[str, Any],
    indicators: Dict[str, Callable],
    include_in_scoring: bool,
) -> Tuple[float]:
    nb = read_nb(nb_path)

    has_syntax_error = False
    indic_vals = list(np.repeat(np.nan, len(indicators)))
    try:
        for i, indic_name in enumerate(indicators):
            indic_vals[i] = round(indicators[indic_name](nb), conf["precision"])
    except SyntaxError as se:
        if conf["print_syntax_errors"]:
            print(f"Syntax error in notebook: {nb_path} reason: ", se)
        has_syntax_error = True
    indic_vals.append(has_syntax_error)
    indic_vals.append(include_in_scoring)
    indic_vals.insert(0, spec_name)

    return tuple(indic_vals)

# %% ../nbs/scilint.ipynb 21
def _get_excluded_paths(paths: Iterable[Path], exclude_pattern: str) -> Iterable[Path]:
    """Excluded paths should either be absolute paths or paths rooted at the project root directory"""
    excl_paths = []
    paths = [p.absolute() for p in paths]

    for ex_pattern in exclude_pattern.split(","):
        if Path(ex_pattern).is_absolute():
            ex_path = Path(ex_pattern)
        else:
            ex_path = Path(get_project_root(), ex_pattern)

        if ex_path.exists():
            excl_paths.extend([p for p in paths if ex_pattern in str(p)])
        elif not ex_path.exists():
            raise ValueError(f"Path component: {ex_path} does not exist")
        else:
            raise ValueError(
                f"Invalid exclusion pattern: {ex_path} pattern is comma separrated list of 'dir/' for directories and 'name.ipynb' for specific notebook"
            )
    return excl_paths

# %% ../nbs/scilint.ipynb 24
def _calculate_warnings(
    spec_name: str,
    scoring_report: pd.DataFrame,
    conf: Dict[str, Any],
    include_missing: bool = False,
) -> Tuple[Dict[str, Any], int]:
    warning_details = []

    for op_text in list(conf["warnings"].keys()):
        for indic in conf["warnings"][op_text]:
            metric_series = scoring_report[indic]
            or_exp = pd.isnull(metric_series) if include_missing else False
            op = (
                operator.lt
                if op_text == "lt"
                else operator.gt
                if op_text == "gt"
                else operator.eq
            )
            warning_data = metric_series[
                (op(metric_series, conf["warnings"][op_text][indic])) | (or_exp)
            ]
            warning_dict = warning_data.to_dict()
            for key, val in warning_dict.items():
                warning_dict[key] = (
                    indic,
                    val,
                    op_text,
                    conf["warnings"][op_text][indic],
                )
            warning_details.append(warning_dict)

    all_warns = _reshape_warnings(spec_name, scoring_report, warning_details)
    num_warnings = len(all_warns)
    return all_warns, num_warnings

# %% ../nbs/scilint.ipynb 27
def _reshape_warnings(
    spec_name: str, scoring_report: pd.DataFrame, warning_details: Iterable[Any]
) -> Dict[str, Iterable[Tuple]]:
    warnings_by_nb = {nb: [] for nb in scoring_report.index}
    for nb in scoring_report.index:
        for wd in warning_details:
            if nb in wd:
                warnings_by_nb[nb].append(tuple([spec_name, nb] + list(wd[nb])))
    warnings_by_nb = {key: val for key, val in warnings_by_nb.items() if len(val) > 0}
    flattened_warns = [item for sublist in warnings_by_nb.values() for item in sublist]
    return pd.DataFrame.from_records(
        data=flattened_warns,
        columns=[
            "spec_name",
            "notebook",
            "indicator",
            "value",
            "operator",
            "threshold",
        ],
    )

# %% ../nbs/scilint.ipynb 30
def lint_nbs(
    spec_name: str,
    conf: Dict[str, Any],
    indicators: Dict[str, Callable],
    nb_paths: Iterable[Path] = None,
    nb_glob: Path = None,
):
    if nb_paths is None:
        nb_paths = [Path(p).absolute() for p in nbglob(nb_glob)]
    else:
        nb_paths = [Path(p).absolute() for p in nb_paths]

    if len(nb_paths) == 0:
        return None, None, None

    excluded_paths = None
    exclusions = conf["exclusions"]
    if exclusions is not None:
        excluded_paths = _get_excluded_paths(nb_paths, exclude_pattern=exclusions)

    results = []
    nb_names = []
    for nb_path in nb_paths:
        include_in_scoring = True
        if exclusions is not None:
            include_in_scoring = False if nb_path in excluded_paths else True

        nb_names.append(nb_path.stem)
        lint_result = lint_nb(spec_name, nb_path, conf, indicators, include_in_scoring)
        results.append(lint_result)

    lint_report = pd.DataFrame.from_records(
        data=results,
        index=nb_names,
        columns=["spec_name"]
        + list(indicators.keys())
        + ["has_syntax_error", "include_in_scoring"],
    ).sort_values(["in_func_pct", "markdown_code_pct"], ascending=False)

    scoring_report = lint_report[lint_report.include_in_scoring].copy()
    all_warns, num_warnings = _calculate_warnings(spec_name, scoring_report, conf)
    return lint_report, all_warns, num_warnings

# %% ../nbs/scilint.ipynb 32
def _map_paths_specs(nb_glob: Path = None, specs_glob: Path = Path(".").resolve()):
    nbs = nbglob(nb_glob)
    spec_files = [
        Path(p)
        for p in globtastic(
            specs_glob,
            file_glob="scilint-*.yaml",
            skip_folder_re="ipynb_checkpoints|_proc",
        )
    ]
    default_spec_files = [p for p in spec_files if p.name == "scilint-default.yaml"]
    default_spec_file = default_spec_files[0] if len(default_spec_files) > 0 else None
    spec_dirs = [p.parent for p in spec_files]

    spec_nbs = {k: [] for k in spec_files}
    for nb in [Path(p) for p in nbs]:
        found_spec = False
        for name, spec_dir in zip(spec_files, spec_dirs):
            if nb.parent == spec_dir:
                spec_nbs[name].append(nb)
                found_spec = True
        if not found_spec:
            if default_spec_file is not None:
                spec_nbs[default_spec_file].append(nb)
            else:
                # Special case: not actually a valid file path - triggers loading a fallback
                fallback_path = Path("scilint-default")
                if fallback_path not in spec_nbs:
                    spec_nbs[fallback_path] = []
                spec_nbs[fallback_path].append(nb)

    return spec_nbs

# %% ../nbs/scilint.ipynb 47
def display_warning_report(all_warns: pd.DataFrame):
    print(
        "\n******************************************Begin Scilint Warning Report*****************************************"
    )
    print(all_warns.to_markdown(tablefmt="grid", index=False))
    print(
        "\n******************************************End Scilint Warning Report*******************************************\n"
    )

# %% ../nbs/scilint.ipynb 49
def _persist_results(
    lint_report: pd.DataFrame, all_warns: pd.DataFrame, conf: Dict[str, Any]
):
    out_dir = Path(conf["out_dir"])
    conf_to_persist = {k: v for k, v in conf.items() if k != "indicators"}
    if not out_dir.exists():
        Path(out_dir).mkdir()
    with open(Path(out_dir, "scilint_config.json"), "w") as outfile:
        json.dump(conf_to_persist, outfile)
    all_warns.to_csv(Path(out_dir, "scilint_warnings.csv"), index=False)
    lint_report.to_csv(Path(out_dir, "scilint_report.csv"))

# %% ../nbs/scilint.ipynb 52
def _load_conf(
    conf_path: str = None,
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    if conf_path is None:
        project_root = find_project_root(tuple([str(Path(".").resolve())]))
        conf_path = Path(project_root, "nbs", "scilint-default.yaml")
        print(f"Loading default lint config: {conf_path}")
    else:
        conf_path = Path(conf_path)

    conf = yaml.safe_load(conf_path.read_text())
    override_names = (
        "exclusions",
        "fail_over",
        "out_dir",
        "precision",
        "print_syntax_errors",
    )
    overrides = (exclusions, fail_over, out_dir, precision, print_syntax_errors)
    for override in zip(override_names, overrides):
        if override[1] is not None:
            conf[override[0]] = override[1]
    return conf

# %% ../nbs/scilint.ipynb 56
def lint(
    display_report: bool = True,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    spec_nbs = _map_paths_specs(nb_glob, specs_glob)
    lint_reports = []
    all_warns = []
    warns_count = []
    for spec, nbs in spec_nbs.items():
        if spec == "scilint-default":
            conf = get_default_spec()
        else:
            conf = _load_conf(
                spec, exclusions, fail_over, out_dir, precision, print_syntax_errors
            )
        if conf["evaluate"] == False:
            print(f"Linting skipped for: {spec.name} as evaluate is set to false")
            continue
        lint_report, report_warns, num_warnings = lint_nbs(
            spec.name, conf, indicator_funcs, nb_paths=nbs
        )
        lint_reports.append(lint_report)
        all_warns.append(report_warns)
        warns_count.append(num_warnings)

        fail_over_conf = conf["fail_over"]
        if conf["fail_over"] == -1:
            print(f"Linting warnings ignored for: {spec.name} as fail_over set to -1")
        elif num_warnings == 0:
            print(f"Linting success for: {spec.name}, no issues found")
        elif num_warnings <= conf["fail_over"]:
            print(
                f"Linting success for: {spec.name}, warnings ({num_warnings}) <= than threshold ({fail_over_conf}) "
            )
        else:
            print(
                f"Linting failed for: {spec.name}, total warnings ({num_warnings}) exceeded threshold ({fail_over_conf})"
            )
            sys.exit(num_warnings)

    lint_report = pd.concat(lint_reports) if len(lint_reports) > 0 else lint_report
    all_warns = pd.concat(all_warns) if len(all_warns) > 0 else report_warns
    num_warnings = sum(warns_count)

    if num_warnings > 0:
        print(
            f"{num_warnings} warnings founds, within tolerated thresholds for all specs"
        )
        if display_report:
            display_warning_report(all_warns)
    elif num_warnings == 0:
        print("No issues found during linting")

    _persist_results(lint_report, all_warns, conf)
    print("Linting completed")

# %% ../nbs/scilint.ipynb 59
def build(
    display_report: bool = True,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    print("Tidying notebooks..")
    tidy()
    if is_nbdev_project():
        nbdev_export.__wrapped__()
        print("Converted notebooks to modules")
        print("Testing notebooks..")
        nbdev_test.__wrapped__()
    print("Running notebook linter..")
    lint(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )
    if is_nbdev_project():
        nbdev_clean.__wrapped__()
        print("Cleaned notebooks")

# %% ../nbs/scilint.ipynb 62
@call_parse
def scilint_tidy():
    tidy()

# %% ../nbs/scilint.ipynb 64
@call_parse
def scilint_lint(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    lint(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

# %% ../nbs/scilint.ipynb 67
@call_parse
def scilint_build(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    build(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

# %% ../nbs/scilint.ipynb 69
@call_parse
def scilint_ci(
    display_report: Param("Print the lint report", store_false) = False,
    nb_glob: Path = None,
    specs_glob: Path = Path(".").resolve(),
    exclusions: str = None,
    fail_over: int = None,
    out_dir: int = None,
    precision: int = None,
    print_syntax_errors: bool = None,
):
    if not is_nbdev_project():
        print("scilint_ci feature is only available for nbdev projects")
        return

    build(
        display_report,
        nb_glob,
        specs_glob,
        exclusions,
        fail_over,
        out_dir,
        precision,
        print_syntax_errors,
    )

    if not shutil.which("quarto"):
        print(
            "Quarto is not installed. A working quarto install is required for the CI build"
        )
        sys.exit(-1)
    nbdev_readme.__wrapped__()
    nbdev_docs.__wrapped__()
