# Copyright (C) 2018-2026  The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information

from collections import defaultdict
import copy
from typing import Any, Dict, List, Tuple

from django.http.request import HttpRequest
from django.http.response import HttpResponse
from django.shortcuts import render
from django.views.decorators.cache import cache_page
from django.views.decorators.clickjacking import xframe_options_exempt

from swh.scheduler.model import SchedulerMetrics
from swh.web.config import scheduler
from swh.web.utils import reverse
from swh.web.utils.archive import count_origins

# Current coverage list of the archive in a high level overview fashion,
# categorized as follow:
#   - listed origins: origins discovered using a swh lister
#   - legacy: origins where public hosting service has closed
#   - deposited: origins coming from swh-deposit
#
# TODO: Store that list in a database table somewhere (swh-scheduler, swh-storage ?)
#       and retrieve it dynamically
listed_origins: Dict[str, Any] = {
    "info": (
        "These software origins get continuously discovered and archived using "
        'the <a href="https://docs.softwareheritage.org/user/listers.html" target="_blank" '
        'rel="noopener noreferrer">listers</a> implemented by Software Heritage.'
    ),
    "origins": [
        {
            "type": "arch",
            "info_url": "https://archlinux.org/",
            "info": "source packages from the Arch Linux distribution",
            "search_pattern": {
                "default": "https://archlinux.org/packages/",
            },
        },
        {
            "type": "aur",
            "info_url": "https://aur.archlinux.org/",
            "info": "source packages from the Arch Linux User Repository",
            "search_pattern": {
                "default": "https://aur.archlinux.org/packages/",
            },
        },
        {
            "type": "bioconductor",
            "info_url": "https://bioconductor.org/",
            "info": "source packages from the Bioconductor project",
            "search_pattern": {
                "default": "https://bioconductor.org/",
            },
        },
        {
            "type": "bitbucket",
            "info_url": "https://bitbucket.org",
            "info": "public repositories from Bitbucket",
            "search_pattern": {
                "default": "https://bitbucket.org/",
            },
        },
        {
            "type": "bower",
            "info_url": "https://bower.io/",
            "info": (
                "git repositories of packages referenced by Bower, "
                "a package manager for the web"
            ),
            "search_pattern": {
                "default": "",
            },
        },
        {
            "type": "cgit",
            "info_url": "https://git.zx2c4.com/cgit/about",
            "info": "public repositories from cgit instances",
            "search_pattern": {
                "default": "cgit",
            },
        },
        {
            "type": "conda",
            "info_url": "https://conda.io/",
            "info": (
                "source packages from the Conda open source package management "
                "system and environment management"
            ),
            "search_pattern": {
                "default": "https://anaconda.org/",
            },
        },
        {
            "type": "cpan",
            "info_url": "https://www.cpan.org/",
            "info": "source packages from The Comprehensive Perl Archive Network",
            "search_pattern": {
                "default": "https://metacpan.org/",
            },
        },
        {
            "type": "cran",
            "info_url": "https://cran.r-project.org",
            "info": "source packages from The Comprehensive R Archive Network",
            "search_pattern": {
                "default": "https://cran.r-project.org/",
            },
        },
        {
            "type": "crates",
            "info_url": "https://crates.io/",
            "info": "source packages from The Rust community's crate registry",
            "search_pattern": {
                "default": "https://crates.io/crates/",
            },
        },
        {
            "type": "dlang",
            "info_url": "https://dlang.org/",
            "info": "public repositories of packages for the D programming language",
            "search_pattern": {
                "default": "",
            },
        },
        {
            "type": "debian",
            "info_url": "https://www.debian.org",
            "info": "source packages from Debian and Debian-based distributions",
            "search_pattern": {
                "default": "deb://",
            },
        },
        {
            "type": "forgejo",
            "info_url": "https://forgejo.org/",
            "info": "public repositories from Forgejo instances",
            "search_pattern": {
                "default": "forgejo",
            },
        },
        {
            "type": "gitea",
            "info_url": "https://about.gitea.com/",
            "info": "public repositories from Gitea instances",
            "search_pattern": {
                "default": "gitea",
            },
        },
        {
            "type": "github",
            "info_url": "https://github.com",
            "info": "public repositories from GitHub",
            "search_pattern": {
                "default": "https://github.com/",
            },
        },
        {
            "type": "gitiles",
            "info_url": "https://github.com/google/gitiles",
            "info": "public repositories from Gitiles instances",
            "search_pattern": {
                "default": "gitiles",
            },
        },
        {
            "type": "gitlab",
            "info_url": "https://gitlab.com",
            "info": "public repositories from multiple GitLab instances",
            "search_pattern": {
                "default": "gitlab",
            },
        },
        {
            "type": "gitweb",
            "info_url": "https://git-scm.com/book/en/v2/Git-on-the-Server-GitWeb",
            "info": "public repositories from GitWeb instances",
            "search_pattern": {
                "default": "gitweb",
            },
        },
        {
            "type": "gogs",
            "info_url": "https://gogs.io/",
            "info": "public repositories from multiple Gogs instances",
            "search_pattern": {
                "default": "gogs",
            },
        },
        {
            "type": "golang",
            "info_url": "https://go.dev/",
            "info": "source packages from the official Go package manager",
            "search_pattern": {
                "default": "https://pkg.go.dev/",
            },
        },
        {
            "type": "guix",
            "info_url": "https://guix.gnu.org",
            "info": "source code tarballs used to build the Guix package collection",
            "main_instance": "guix.gnu.org",
            "search_pattern": {
                "default": "",
            },
        },
        {
            "type": "GNU",
            "info_url": "https://www.gnu.org",
            "info": "releases from the GNU project (as of August 2015)",
            "search_pattern": {
                "default": "gnu",
            },
        },
        {
            "type": "hackage",
            "info_url": "https://hackage.haskell.org",
            "info": "source packages from The Haskell Package Repository",
            "search_pattern": {
                "default": "https://hackage.haskell.org/package/",
            },
        },
        {
            "type": "heptapod",
            "info_url": "https://heptapod.net/",
            "info": "public repositories from multiple Heptapod instances",
            "search_pattern": {
                "default": "heptapod",
            },
        },
        {
            "type": "launchpad",
            "info_url": "https://launchpad.net",
            "logo": "img/logos/launchpad.png",
            "info": "public repositories from Launchpad",
            "search_pattern": {
                "default": "launchpad.net/",
            },
        },
        {
            "type": "maven",
            "info_url": "https://maven.apache.org/",
            "info": "java source packages from maven repositories",
            "search_pattern": {
                "default": "maven",
                "cvs": "",
                "git": "",
                "hg": "",
                "svn": "",
            },
        },
        {
            "type": "nixos",
            "info_url": "https://nixos.org",
            "info": "source code tarballs and patches used to build the Nix package collection",
            "main_instance": "nixpkgs-swh.nixos.org",
            "search_pattern": {
                "default": "",
                "content": "https://cache.nixos.org",
                "tarball-directory": "https://cache.nixos.org",
            },
        },
        {
            "type": "npm",
            "info_url": "https://www.npmjs.com",
            "info": "public packages from the package registry for javascript",
            "search_pattern": {
                "default": "https://www.npmjs.com",
            },
        },
        {
            "type": "opam",
            "info_url": "https://opam.ocaml.org/",
            "info": "public packages from the source-based package manager for OCaml",
            "search_pattern": {
                "default": "opam+https://",
            },
        },
        {
            "type": "Packagist",
            "info_url": "https://packagist.org/",
            "info": "source code repositories referenced by The PHP Package Repository",
            "search_pattern": {
                "default": "",
            },
        },
        {
            "type": "pagure",
            "info_url": "https://pagure.io/pagure",
            "info": "public repositories from multiple Pagure instances",
            "search_pattern": {
                "default": "",
            },
        },
        {
            "type": "phabricator",
            "info_url": "https://www.phacility.com/phabricator",
            "info": "public repositories from multiple Phabricator instances",
            "search_pattern": {
                "default": "phabricator",
            },
        },
        {
            "type": "pubdev",
            "info_url": "https://pub.dev",
            "info": "source packages from the official repository for Dart and Flutter apps",
            "search_pattern": {
                "default": "https://pub.dev",
            },
        },
        {
            "type": "puppet",
            "info_url": "https://forge.puppet.com/",
            "info": "source packages from the Puppet Forge",
            "search_pattern": {
                "default": "https://forge.puppet.com/modules/",
            },
        },
        {
            "type": "pypi",
            "info_url": "https://pypi.org",
            "info": "source packages from the Python Package Index",
            "search_pattern": {
                "default": "https://pypi.org",
            },
        },
        {
            "type": "rpm",
            "info_url": "https://www.redhat.com",
            "info": "source packages from Red Hat based distributions",
            "search_pattern": {
                "default": "rpm",
            },
        },
        {
            "type": "rubygems",
            "info_url": "https://rubygems.org",
            "info": "source packages from the Ruby community's gem hosting service",
            "search_pattern": {
                "default": "https://rubygems.org/gems/",
            },
        },
        {
            "type": "sourceforge",
            "info_url": "https://sourceforge.net",
            "info": "public repositories from SourceForge",
            "search_pattern": {
                "default": "code.sf.net",
                "bzr": "bzr.sourceforge.net",
                "cvs": "cvs.sourceforge.net",
            },
        },
        {
            "type": "stagit",
            "info_url": "https://codemadness.org/stagit.html",
            "info": "public repositories from Stagit instances",
            "search_pattern": {
                "default": "stagit",
            },
        },
    ],
}

legacy_origins: Dict[str, Any] = {
    "info": (
        "Discontinued hosting services. Those origins have been archived "
        "by Software Heritage."
    ),
    "origins": [
        {
            "type": "bitbucket-hg",
            "info_url": "https://bitbucket.org",
            "info": "public mercurial repositories from Bitbucket",
            "search_pattern": "https://bitbucket.org/",
            "visit_types": ["hg"],
        },
        {
            "type": "gitorious",
            "info_url": "https://en.wikipedia.org/wiki/Gitorious",
            "info": (
                "public repositories from the former Gitorious code hosting service"
            ),
            "visit_types": ["git"],
            "search_pattern": "https://gitorious.org",
        },
        {
            "type": "googlecode",
            "info_url": "https://code.google.com/archive",
            "info": (
                "public repositories from the former Google Code project "
                "hosting service"
            ),
            "visit_types": ["git", "hg", "svn"],
            "search_pattern": "googlecode.com",
        },
        {
            "type": "osdn",
            "info_url": "https://en.wikipedia.org/wiki/OSDN",
            "info": (
                "public repositories from the former Open Source Development Network "
                "hosting service"
            ),
            "visit_types": ["cvs", "git", "hg", "svn"],
            "search_pattern": "osdn.net",
        },
    ],
}

deposited_origins: Dict[str, Any] = {
    "info": (
        "These origins are directly pushed into the archive by trusted partners "
        "using the "
        '<a href="https://docs.softwareheritage.org/devel/architecture/overview.html#deposit" '
        'target="_blank" rel="noopener noreferrer">deposit</a> service of Software Heritage.'
    ),
    "origins": [
        {
            "type": "elife",
            "info_url": "https://elifesciences.org",
            "info": (
                "research software source code associated to the articles "
                "eLife publishes"
            ),
            "search_pattern": "elife.stencila.io",
            "visit_types": ["deposit"],
        },
        {
            "type": "hal",
            "info_url": "https://hal.archives-ouvertes.fr",
            "info": "scientific software source code deposited in the open archive HAL",
            "visit_types": ["deposit"],
            "search_pattern": "hal.archives-ouvertes.fr",
        },
        {
            "type": "ipol",
            "info_url": "https://www.ipol.im",
            "info": "software artifacts associated to the articles IPOL publishes",
            "visit_types": ["deposit"],
            "search_pattern": "doi.org/10.5201/ipol",
        },
        {
            "type": "zenodo",
            "info_url": "https://zenodo.org/",
            "info": "software source code deposited in the Open Science platform Zenodo",
            "visit_types": ["deposit"],
            "search_pattern": "doi.org/10.5281/zenodo",
        },
    ],
}


def get_listers_metrics() -> Dict[str, List[Tuple[str, SchedulerMetrics]]]:
    """Returns scheduler metrics in the following mapping:
    Dict[lister_name, List[Tuple[instance_name, SchedulerMetrics]]]
    as a lister instance has one SchedulerMetrics object per visit type.
    """
    listers_metrics = defaultdict(list)
    listers = scheduler().get_listers()
    scheduler_metrics = scheduler().get_metrics()
    for lister in listers:
        for metrics in filter(lambda m: m.lister_id == lister.id, scheduler_metrics):
            listers_metrics[lister.name].append((lister.instance_name, metrics))

    return listers_metrics


def _search_url(query: str, visit_type: str) -> str:
    return reverse(
        "browse-search",
        query_params={
            "q": query,
            "visit_type": visit_type,
            "with_visit": "true",
            "with_content": "true",
        },
    )


@xframe_options_exempt
@cache_page(24 * 60 * 60)  # one day
def swh_coverage(request: HttpRequest) -> HttpResponse:
    listers_metrics = get_listers_metrics()

    for origins in listed_origins["origins"]:
        origins["count"] = "0"
        origins["instances"] = {}
        origins_type = origins["type"]

        # special processing for nixos/guix origins
        if origins_type in ("nixos", "guix"):
            main_instance = origins["main_instance"]
            visit_data = listers_metrics.get("nixguix", [])
            visit_type_counts: Dict[str, Dict[str, Dict[str, str]]] = defaultdict(dict)
            # visit types from new nixguix lister
            count = 0
            for instance, metrics in visit_data:
                if instance != main_instance:
                    continue
                instance_type_count = (
                    metrics.origins_enabled - metrics.origins_never_visited
                )
                if instance_type_count > 0:
                    visit_type_counts[instance][metrics.visit_type] = {
                        "count": f"{instance_type_count:,}"
                    }

                count += instance_type_count

            origins["count"] = f"{count:,}"
            origins["instances"] = dict(visit_type_counts)

        if origins_type not in listers_metrics:
            continue

        count_total = sum(
            [metrics.origins_enabled for _, metrics in listers_metrics[origins_type]]
        )
        count_never_visited = sum(
            [
                metrics.origins_never_visited
                for _, metrics in listers_metrics[origins_type]
            ]
        )
        count_visited = count_total - count_never_visited

        origins["count"] = f"{count_visited:,}"
        origins["instances"] = defaultdict(dict)
        for instance, metrics in listers_metrics[origins_type]:
            instance_count = metrics.origins_enabled - metrics.origins_never_visited
            # no archived origins for that visit type, skip it
            if instance_count == 0:
                continue

            origins["instances"][instance].update(
                {metrics.visit_type: {"count": f"{instance_count:,}"}}
            )
            origins["visit_types"] = list(
                set(origins["instances"][instance].keys())
                | set(origins.get("visit_types", []))
            )

        # defaultdict cannot be iterated in django template
        origins["instances"] = dict(sorted(origins["instances"].items()))

    for origins in listed_origins["origins"]:
        instances = origins["instances"]
        nb_instances = len(instances)
        for instance_name, visit_types in instances.items():
            for visit_type in visit_types:
                search_url = ""
                if visit_type in origins["search_pattern"]:
                    search_pattern = origins["search_pattern"][visit_type]
                elif nb_instances > 1:
                    search_pattern = instance_name
                else:
                    search_pattern = origins["search_pattern"]["default"]
                if search_pattern:
                    search_url = _search_url(search_pattern, visit_type)
                visit_types[visit_type]["search_url"] = search_url

    # use a light copy to avoid side effects when running tests
    listed_origins_filtered = copy.copy(listed_origins)
    # filter out origin types without archived origins
    listed_origins_filtered["origins"] = list(
        filter(lambda o: o["count"] != "0", listed_origins_filtered["origins"])
    )

    for origins in legacy_origins["origins"]:
        origins["instances"] = {origins["type"]: {}}
        total_count = 0
        for visit_type in origins["visit_types"]:
            count = count_origins(
                origins["search_pattern"], with_visit=True, visit_types=[visit_type]
            )
            total_count += count
            origins["instances"][origins["type"]][visit_type] = {
                "count": f"{count:,}",
                "search_url": _search_url(origins["search_pattern"], visit_type),
            }
        origins["count"] = f"{total_count:,}"

    for origins in deposited_origins["origins"]:
        origins_count = count_origins(
            origins["search_pattern"], with_visit=True, visit_types=["deposit"]
        )
        origins["count"] = f"{origins_count:,}"
        origins["search_urls"] = {
            "deposit": _search_url(origins["search_pattern"], "deposit")
        }

    focus = []
    focus_param = request.GET.get("focus")
    if focus_param:
        focus = focus_param.split(",")

    return render(
        request,
        "archive-coverage.html",
        {
            "origins": {
                "Regular crawling": listed_origins_filtered,
                "Discontinued hosting": legacy_origins,
                "On demand archival": deposited_origins,
            },
            "focus": focus,
        },
    )
