# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/scilint.ipynb.

# %% auto 0
__all__ = ['run_nbqa_cmd', 'tidy', 'scilint_tidy', 'is_nbdev_project', 'get_func_defs', 'count_func_calls',
           'replace_ipython_magics', 'safe_div', 'get_cell_code', 'calls_per_func', 'mean_cpf', 'median_cpf', 'afr',
           'count_inline_asserts', 'iaf', 'mean_iaf', 'median_iaf', 'calc_ifp', 'ifp', 'mcp', 'tcl', 'lint_nb',
           'format_quality_warning', 'get_excluded_paths', 'lint_nbs', 'calculate_warnings', 'scilint_lint',
           'scilint_build', 'scilint_ci']

# %% ../nbs/scilint.ipynb 2
import ast
import os
import re
import shutil
import sys
import warnings
from collections import Counter
from configparser import InterpolationMissingOptionError
from pathlib import Path
from typing import Iterable

import nbformat
import numpy as np
import pandas as pd
from execnb.nbio import read_nb
from fastcore.script import call_parse
from nbdev.clean import nbdev_clean
from nbdev.config import get_config
from nbdev.doclinks import nbdev_export, nbglob
from nbdev.quarto import nbdev_docs, nbdev_readme
from nbdev.test import nbdev_test
from nbqa.__main__ import _get_configs, _main
from nbqa.cmdline import CLIArgs
from nbqa.find_root import find_project_root

# %% ../nbs/scilint.ipynb 7
def run_nbqa_cmd(cmd):
    print(f"Running {cmd}")
    project_root: Path = find_project_root(tuple([str(Path(".").resolve())]))
    args = CLIArgs.parse_args([cmd, str(project_root)])
    configs = _get_configs(args, project_root)
    output_code = _main(args, configs)
    return output_code

# %% ../nbs/scilint.ipynb 9
def tidy():
    tidy_tools = ["black", "isort", "autoflake"]
    [run_nbqa_cmd(c) for c in tidy_tools]

# %% ../nbs/scilint.ipynb 10
@call_parse
def scilint_tidy():
    tidy()

# %% ../nbs/scilint.ipynb 12
def is_nbdev_project(project_path: Path = Path(".")):
    is_nbdev = True
    project_root = find_project_root(tuple([str(project_path.resolve())]))

    if not Path(project_root, "settings.ini").exists():
        is_nbdev = False
    try:
        get_config().lib_name
    except InterpolationMissingOptionError:
        is_nbdev = False

    return is_nbdev

# %% ../nbs/scilint.ipynb 17
def get_func_defs(code, ignore_private_prefix=True):
    func_names = []
    for stmt in ast.walk(ast.parse(code)):
        if isinstance(stmt, ast.FunctionDef):
            inner_cond = (
                False if ignore_private_prefix and stmt.name.startswith("_") else True
            )
            if inner_cond:
                func_names.append(stmt.name)
    return func_names

# %% ../nbs/scilint.ipynb 19
def count_func_calls(code, func_defs):
    func_calls = Counter({k: 0 for k in func_defs})
    for stmt in ast.walk(ast.parse(code)):
        if isinstance(stmt, ast.Call):
            func_name = stmt.func.id if "id" in stmt.func.__dict__ else stmt.func.attr
            if func_name in func_defs:
                if func_name in func_calls:
                    func_calls[func_name] += 1
    return func_calls

# %% ../nbs/scilint.ipynb 23
def replace_ipython_magics(code):
    # Replace Ipython magic and shell command symbol with comment
    code = code.replace("%", "#")
    code = re.sub(r"^!", "#", code)
    return re.sub(r"\n\W?!", "\n#", code)

# %% ../nbs/scilint.ipynb 25
def safe_div(numer, denom):
    return 0 if denom == 0 else numer / denom

# %% ../nbs/scilint.ipynb 27
def get_cell_code(nb):
    pnb = nbformat.from_dict(nb)
    nb_cell_code = "\n".join(
        [
            replace_ipython_magics(c["source"])
            for c in pnb.cells
            if c["cell_type"] == "code"
        ]
    )
    return nb_cell_code

# %% ../nbs/scilint.ipynb 29
def calls_per_func(nb):
    nb_cell_code = get_cell_code(nb)
    func_defs = get_func_defs(nb_cell_code)
    func_calls = count_func_calls(nb_cell_code, func_defs)
    return func_calls

# %% ../nbs/scilint.ipynb 30
def mean_cpf(nb):
    return pd.Series(calls_per_func(nb)).mean()

# %% ../nbs/scilint.ipynb 31
def median_cpf(nb):
    with warnings.catch_warnings():
        warnings.filterwarnings(action="ignore", message="Mean of empty slice")
        return pd.Series(calls_per_func(nb)).median()

# %% ../nbs/scilint.ipynb 39
def afr(nb):
    nb_cell_code = get_cell_code(nb)
    if nb_cell_code == "":  # no code cells - metric is not well defined
        return np.nan
    func_defs = get_func_defs(nb_cell_code)
    num_funcs = len(func_defs)

    assert_count = 0
    for stmt in ast.walk(ast.parse(nb_cell_code)):
        if isinstance(stmt, ast.Assert):
            assert_count += 1

    return safe_div(assert_count, num_funcs)

# %% ../nbs/scilint.ipynb 42
def count_inline_asserts(code, func_defs):
    inline_func_asserts = Counter({k: 0 for k in func_defs})

    for stmt in ast.walk(ast.parse(code)):
        if isinstance(stmt, ast.Assert):
            for assert_st in ast.walk(stmt):
                if isinstance(assert_st, ast.Call):
                    func_name = (
                        assert_st.func.id
                        if "id" in assert_st.func.__dict__
                        else assert_st.func.attr
                    )
                    if func_name in inline_func_asserts:
                        inline_func_asserts[func_name] += 1
    return inline_func_asserts

# %% ../nbs/scilint.ipynb 43
def iaf(nb):
    nb_cell_code = get_cell_code(nb)
    if nb_cell_code == "":
        return np.nan
    func_defs = get_func_defs(nb_cell_code)
    return count_inline_asserts(nb_cell_code, func_defs)

# %% ../nbs/scilint.ipynb 50
def mean_iaf(nb):
    return pd.Series(iaf(nb)).mean()

# %% ../nbs/scilint.ipynb 51
def median_iaf(nb):
    with warnings.catch_warnings():
        warnings.filterwarnings(action="ignore", message="Mean of empty slice")
        return pd.Series(iaf(nb)).median()

# %% ../nbs/scilint.ipynb 55
def calc_ifp(nb_cell_code):
    stmts_in_func = 0
    stmts_outside_func = 0
    for stmt in ast.walk(ast.parse(replace_ipython_magics(nb_cell_code))):
        if isinstance(stmt, ast.FunctionDef) and not stmt.name.startswith("_"):
            for body_item in stmt.body:
                stmts_in_func += 1
        elif isinstance(stmt, ast.Module):
            for body_item in stmt.body:
                if not isinstance(body_item, ast.FunctionDef):
                    stmts_outside_func += 1
    return (
        0
        if stmts_outside_func + stmts_in_func == 0
        else (stmts_in_func / (stmts_outside_func + stmts_in_func)) * 100
    )

# %% ../nbs/scilint.ipynb 57
def ifp(nb):
    nb_cell_code = "\n".join(
        [
            replace_ipython_magics(c["source"])
            for c in nb.cells
            if c["cell_type"] == "code"
        ]
    )
    if nb_cell_code == "":
        return np.nan
    return calc_ifp(nb_cell_code)

# %% ../nbs/scilint.ipynb 60
def mcp(nb):
    md_cells = [c for c in nb.cells if c["cell_type"] == "markdown"]
    code_cells = [c for c in nb.cells if c["cell_type"] == "code"]
    num_code_cells = len(code_cells)
    if num_code_cells == 0:
        return np.nan
    num_md_cells = len(md_cells)
    return (
        100
        if num_code_cells == 0
        else (num_md_cells / (num_md_cells + num_code_cells)) * 100
    )

# %% ../nbs/scilint.ipynb 63
def tcl(nb):
    return sum([len(c["source"]) for c in nb.cells if c["cell_type"] == "code"])

# %% ../nbs/scilint.ipynb 65
def lint_nb(
    nb_path, include_in_scoring, rounding_precision=3, print_syntax_errors=False
):
    nb = read_nb(nb_path)

    nb_cpf_median = np.nan
    nb_cpf_mean = np.nan
    nb_ifp = np.nan
    nb_afr = np.nan
    nb_iaf_median = np.nan
    nb_iaf_mean = np.nan
    nb_mcp = np.nan
    nb_tcl = np.nan
    has_syntax_error = False

    try:
        nb_cpf_median = round(median_cpf(nb), rounding_precision)
        nb_cpf_mean = round(mean_cpf(nb), rounding_precision)
        nb_ifp = round(ifp(nb), rounding_precision)
        nb_afr = round(afr(nb), rounding_precision)
        nb_iaf_median = round(median_iaf(nb), rounding_precision)
        nb_iaf_mean = round(mean_iaf(nb), rounding_precision)
        nb_mcp = round(mcp(nb), rounding_precision)
        nb_tcl = round(tcl(nb), rounding_precision)
    except SyntaxError as se:
        if print_syntax_errors:
            print(f"Syntax error in notebook: {nb_path} reason: ", se)
        has_syntax_error = True

    return (
        nb_cpf_median,
        nb_cpf_mean,
        nb_ifp,
        nb_afr,
        nb_iaf_median,
        nb_iaf_mean,
        nb_mcp,
        nb_tcl,
        include_in_scoring,
        has_syntax_error,
    )

# %% ../nbs/scilint.ipynb 66
# TODO generate and persist a new dataframe of warnings from this..


def format_quality_warning(metric, warning_data, warn_thresh, direction):
    for warning_row in warning_data.reset_index().itertuples():
        print(f'"{warning_row.index}" has: {metric} {direction} {warn_thresh}')

# %% ../nbs/scilint.ipynb 67
def get_excluded_paths(paths: Iterable[Path], exclude_pattern: str):
    excl_paths = []
    for ex_pattern in exclude_pattern.split(","):
        ex_path = Path(ex_pattern)
        if ex_path.exists():
            excl_paths.extend([p for p in paths if ex_pattern in str(p)])
        elif not ex_path.exists():
            raise ValueError(f"Path component: {ex_path} does not exist")
        else:
            raise ValueError(
                f"Invalid exclusion pattern: {ex_path} pattern is comma separrated list of 'dir/' for directories and 'name.ipynb' for specific notebook"
            )
    return excl_paths

# %% ../nbs/scilint.ipynb 69
def lint_nbs(
    cpf_med_warn_thresh=1,
    cpf_mean_warn_thresh=1,
    ifp_warn_thresh=20,
    afr_warn_thresh=1,
    iaf_med_warn_thresh=0,
    iaf_mean_warn_thresh=0.5,
    mcp_warn_thresh=5,
    tcl_warn_thresh=30000,
    rounding_precision=3,
    csv_out_path="/tmp/scilint.csv",
    exclusions=None,
    print_syntax_errors=False,
):
    nb_paths = [Path(p) for p in nbglob(Path("."))]

    excluded_paths = None
    if exclusions is not None:
        excluded_paths = get_excluded_paths(nb_paths, exclude_pattern=exclusions)

    lt_metric_cols = [
        "cpf_median",
        "cpf_mean",
        "in_function_pct",
        "asserts_function_ratio",
        "iaf_median",
        "iaf_mean",
        "markdown_code_pct",
    ]
    gt_metric_cols = ["total_code_len"]
    lt_metrics_thresholds = [
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
    ]
    gt_metrics_thresholds = [tcl_warn_thresh]
    results = []
    nb_names = []
    for nb_path in nb_paths:
        include_in_scoring = True
        if exclusions is not None:
            include_in_scoring = False if nb_path in excluded_paths else True

        nb_names.append(nb_path.stem)
        lint_result = lint_nb(
            nb_path, include_in_scoring, rounding_precision, print_syntax_errors
        )
        results.append(lint_result)

    lint_report = pd.DataFrame.from_records(
        data=results,
        index=nb_names,
        columns=lt_metric_cols
        + gt_metric_cols
        + ["include_in_scoring", "has_syntax_error"],
    ).sort_values(["cpf_median", "markdown_code_pct"], ascending=False)

    # Calculate warnings only from notebooks included in scoring
    scoring_report = lint_report[lint_report.include_in_scoring].copy()

    num_warnings = calculate_warnings(
        scoring_report,
        lt_metric_cols,
        lt_metrics_thresholds,
        gt_metric_cols,
        gt_metrics_thresholds,
    )

    lint_report.to_csv(csv_out_path)

    return lint_report, num_warnings

# %% ../nbs/scilint.ipynb 70
def calculate_warnings(
    scoring_report,
    lt_metric_cols,
    lt_metrics_thresholds,
    gt_metric_cols,
    gt_metrics_thresholds,
):
    syntax_error_nbs = scoring_report[scoring_report.has_syntax_error].index.to_list()
    # TODO tidy this up to usual config dict
    print("\n*********************Begin Scilint Report*********************")
    num_warnings = 0

    for lt_metric_col, lt_metrics_threshold in zip(
        lt_metric_cols, lt_metrics_thresholds
    ):
        metrics_series = scoring_report[lt_metric_col]
        warning_data = metrics_series[metrics_series < lt_metrics_threshold]

        num_warnings += len(warning_data)
        format_quality_warning(
            lt_metric_col,
            warning_data,
            lt_metrics_threshold,
            direction="<",
        )
    for gt_metric_col, gt_metrics_threshold in zip(
        gt_metric_cols, gt_metrics_thresholds
    ):
        metrics_series = scoring_report[gt_metric_col]
        warning_data = metrics_series[metrics_series > gt_metrics_threshold]

        num_warnings += len(warning_data)
        format_quality_warning(
            gt_metric_col,
            warning_data,
            gt_metrics_threshold,
            direction=">",
        )

    num_warnings += len(syntax_error_nbs)
    for syntax_error_nb in syntax_error_nbs:
        print(f'"{syntax_error_nb}" has syntax errors')

    if num_warnings == 0:
        print("No issues found")
    print("*********************End Scilint Report***********************")
    return num_warnings

# %% ../nbs/scilint.ipynb 77
def _lint(
    cpf_med_warn_thresh: float = 1,
    cpf_mean_warn_thresh: float = 1,
    ifp_warn_thresh: float = 20,
    afr_warn_thresh: float = 1,
    iaf_med_warn_thresh: float = 0,
    iaf_mean_warn_thresh: float = 0.5,
    mcp_warn_thresh: float = 5,
    tcl_warn_thresh: float = 30000,
    rounding_precision: int = 3,
    csv_out_path: str = "/tmp/scilint.csv",
    exclusions: str = None,
    fail_over: int = 1,
    print_syntax_errors: bool = False,
):
    lint_report, num_warnings = lint_nbs(
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
        tcl_warn_thresh,
        rounding_precision,
        csv_out_path,
        exclusions,
        print_syntax_errors,
    )
    if fail_over == -1:
        print("Linting outcome ignored as fail_over set to -1")
    elif num_warnings > fail_over:
        print(
            f"Linting failed: total warnings ({num_warnings}) exceeded threshold ({fail_over})"
        )
        sys.exit(num_warnings)
    else:
        print("Linting succeeded")

# %% ../nbs/scilint.ipynb 78
def _build(
    cpf_med_warn_thresh: float = 1,
    cpf_mean_warn_thresh: float = 1,
    ifp_warn_thresh: float = 20,
    afr_warn_thresh: float = 1,
    iaf_med_warn_thresh: float = 0,
    iaf_mean_warn_thresh: float = 0.5,
    mcp_warn_thresh: float = 5,
    tcl_warn_thresh: float = 30000,
    rounding_precision: int = 3,
    csv_out_path: str = "/tmp/scilint.csv",
    exclusions: str = None,
    fail_over: int = 1,
    print_syntax_errors: bool = False,
):
    print("Tidying notebooks..")
    tidy()
    if is_nbdev_project():
        nbdev_export.__wrapped__()
        print("Converted notebooks")
        print("Testing notebooks..")
        nbdev_test.__wrapped__()
    print("Running notebook linter..")
    _lint(
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
        tcl_warn_thresh,
        rounding_precision,
        csv_out_path,
        exclusions,
        fail_over,
        print_syntax_errors,
    )
    if is_nbdev_project():
        print("Cleaning notebooks..")
        nbdev_clean.__wrapped__()

# %% ../nbs/scilint.ipynb 80
@call_parse
def scilint_lint(
    cpf_med_warn_thresh: float = 1,
    cpf_mean_warn_thresh: float = 1,
    ifp_warn_thresh: float = 20,
    afr_warn_thresh: float = 1,
    iaf_med_warn_thresh: float = 0,
    iaf_mean_warn_thresh: float = 0.5,
    mcp_warn_thresh: float = 5,
    tcl_warn_thresh: float = 30000,
    rounding_precision: int = 3,
    csv_out_path: str = "/tmp/scilint.csv",
    exclusions: str = None,
    fail_over: int = 1,
    print_syntax_errors: bool = False,
):
    _lint(
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
        tcl_warn_thresh,
        rounding_precision,
        csv_out_path,
        exclusions,
        fail_over,
    )

# %% ../nbs/scilint.ipynb 81
@call_parse
def scilint_build(
    cpf_med_warn_thresh: float = 1,
    cpf_mean_warn_thresh: float = 1,
    ifp_warn_thresh: float = 20,
    afr_warn_thresh: float = 1,
    iaf_med_warn_thresh: float = 0,
    iaf_mean_warn_thresh: float = 0.5,
    mcp_warn_thresh: float = 5,
    tcl_warn_thresh: float = 30000,
    rounding_precision: int = 3,
    csv_out_path: str = "/tmp/scilint.csv",
    exclusions: str = None,
    fail_over: int = 1,
):
    _build(
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
        tcl_warn_thresh,
        rounding_precision,
        csv_out_path,
        exclusions,
        fail_over,
    )

# %% ../nbs/scilint.ipynb 82
@call_parse
def scilint_ci(
    cpf_med_warn_thresh: float = 1,
    cpf_mean_warn_thresh: float = 1,
    ifp_warn_thresh: float = 20,
    afr_warn_thresh: float = 1,
    iaf_med_warn_thresh: float = 0,
    iaf_mean_warn_thresh: float = 0.5,
    mcp_warn_thresh: float = 5,
    tcl_warn_thresh: int = 30000,
    rounding_precision: int = 3,
    csv_out_path: str = "/tmp/scilint.csv",
    exclusions: str = None,
    fail_over: int = 1,
):
    if not is_nbdev_project():
        print("scilint_ci feature is only available for nbdev projects")
        return

    _build(
        cpf_med_warn_thresh,
        cpf_mean_warn_thresh,
        ifp_warn_thresh,
        afr_warn_thresh,
        iaf_med_warn_thresh,
        iaf_mean_warn_thresh,
        mcp_warn_thresh,
        tcl_warn_thresh,
        rounding_precision,
        csv_out_path,
        exclusions,
        fail_over,
    )
    if not shutil.which("quarto"):
        print(
            "Quarto is not installed. A working quarto install is required for the CI build"
        )
        sys.exit(-1)
    nbdev_readme.__wrapped__()
    nbdev_docs.__wrapped__()
