#!/usr/bin/env python3
#
# Single-threaded DSpace downloader using OAI-PMH.
# 1 Resolves the input handle to an OAI set
# 2 Harvests item records and ORIGINAL bitstream URLs from METS
# 3 Downloads files sequentially
#
# Usage:
#   ./dspace_wgetter.py <collection_handle_url> [output_dir]
#
# Example:
#   ./dspace_wgetter.py \
#     'https://dspace.nplg.gov.ge/handle/1234/524262' \
#     './sakhalkho_gazeti_download'
#

import argparse
import hashlib
import json
import os
import re
import signal
import shutil
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as etree
from collections.abc import Iterator
from collections import Counter
from pathlib import Path


OAI_NS = {
    'mets': 'http://www.loc.gov/METS/',
    'oai': 'http://www.openarchives.org/OAI/2.0/',
}
XLINK_HREF = '{http://www.w3.org/1999/xlink}href'
HTTP_USER_AGENT = 'dspace_wgetter'
ACTIVE_RUN_LOG: Path | None = None


class OAIError(RuntimeError):
    pass


def handle_sigint(signum: int, frame: object) -> None:
    raise KeyboardInterrupt


def append_text(path: Path, text: str) -> None:
    with path.open('a', encoding='utf-8') as handle:
        handle.write(text)


def log(message: str, run_log: Path) -> None:
    print(message, flush=True)
    append_text(run_log, f'{message}\n')


def log_error(log_path: Path, message: str) -> None:
    timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
    append_text(log_path, f'[{timestamp}] {message}\n')


def write_download_rows(path: Path, rows: list[tuple[str, str, int | None]]) -> None:
    '''Write the download queue manifest atomically.

    `rows` is a list of `(relative_path, url, size_bytes)` tuples:
    - `relative_path`: output path relative to the collection `files/` directory
    - `url`: bitstream URL to download
    - `size_bytes`: remote file size from METS, or `None` if unknown

    The manifest is stored as UTF-8 TSV with one row per file:
    `relative_path<TAB>url<TAB>size_bytes`

    The file is written to a temporary path, flushed, fsynced, and then
    replaced into place so the resume queue is less likely to be corrupted by
    an interruption or power loss.
    '''
    path.parent.mkdir(parents=True, exist_ok=True)
    tmp_path = path.with_name(f'.{path.name}.tmp')
    with tmp_path.open('w', encoding='utf-8') as handle:
        for relative_path, url, size_bytes in rows:
            size_field = '' if size_bytes is None else str(size_bytes)
            handle.write(f'{relative_path}\t{url}\t{size_field}\n')
        handle.flush()
        os.fsync(handle.fileno())
    tmp_path.replace(path)
    dir_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
    try:
        dir_fd = os.open(path.parent, dir_flags)
    except OSError:
        return
    try:
        os.fsync(dir_fd)
    finally:
        os.close(dir_fd)


def remaining_resume_rows(
    failed_rows: list[tuple[str, str, int | None]],
    all_rows: list[tuple[str, str, int | None]],
    processed_count: int,
) -> list[tuple[str, str, int | None]]:
    pending_rows = list(failed_rows)
    pending_rows.extend(all_rows[processed_count:])
    return pending_rows


def read_download_rows(path: Path) -> list[tuple[str, str, int | None]]:
    rows: list[tuple[str, str, int | None]] = []
    with path.open(encoding='utf-8') as handle:
        for line in handle:
            parts = line.rstrip('\n').split('\t')
            if len(parts) < 2 or not parts[1]:
                continue
            size_bytes = None
            if len(parts) >= 3 and parts[2].isdigit():
                size_bytes = int(parts[2])
            rows.append((parts[0], parts[1], size_bytes))
    return rows


def parse_collection_url(url: str) -> tuple[str, str]:
    parsed_url = urllib.parse.urlparse(url)
    if parsed_url.scheme not in {'http', 'https'} or not parsed_url.netloc:
        raise SystemExit('Could not parse collection URL. Expected an http(s) DSpace handle URL.')
    match = re.search(r'(/handle/\d+/\d+)$', parsed_url.path)
    if not match:
        raise SystemExit('Could not parse collection URL. Expected a DSpace handle URL ending in /handle/XXXX/YYYY.')
    return f'{parsed_url.scheme}://{parsed_url.netloc}', match.group(1)


def sanitize_collection_dir_name(name: str) -> str:
    safe = re.sub(r'\s+', '_', name, flags=re.UNICODE)
    safe = safe.replace('/', '_')
    safe = re.sub(r'[^\w.-]', '_', safe, flags=re.UNICODE)
    safe = re.sub(r'_+', '_', safe)
    return safe.strip('_')


def year_folder_from_filename(file_name: str) -> str:
    stem = Path(file_name).stem
    match = re.search(r'(17\d{2}|18\d{2}|19\d{2}|20\d{2})', stem)
    if match:
        return match.group(1)
    return 'unknown_year'


def format_size_mb(size_bytes: int | None) -> str:
    if size_bytes is None:
        return 'unknown'
    return f'{size_bytes / (1024 * 1024):.1f}MB'


def natural_sort_part(part: str) -> tuple[tuple[int, int | str], ...]:
    tokens = re.split(r'(\d+)', part)
    key: list[tuple[int, int | str]] = []
    for token in tokens:
        if not token:
            continue
        if token.isdigit():
            key.append((0, int(token)))
        else:
            key.append((1, token.casefold()))
    return tuple(key)


def natural_relative_path_key(path_text: str) -> tuple[tuple[tuple[int, int | str], ...], ...]:
    return tuple(natural_sort_part(part) for part in Path(path_text).parts)


def extract_file_name_from_url(url: str, fallback_name: str) -> str:
    parsed_url = urllib.parse.urlparse(url)
    file_name = urllib.parse.unquote(Path(parsed_url.path).name)
    file_name = file_name.split('?', 1)[0]
    file_name = file_name.split('#', 1)[0]
    file_name = file_name.replace('/', '_').replace('\\', '_')
    if not file_name:
        return fallback_name
    return file_name


def append_suffix_to_file_name(file_name: str, suffix: str) -> str:
    path = Path(file_name)
    full_suffix = ''.join(path.suffixes)
    stem = file_name[:-len(full_suffix)] if full_suffix else file_name
    return f'{stem}{suffix}{full_suffix}'


def bitstream_identity_suffix(url: str) -> str:
    parsed_url = urllib.parse.urlparse(url)
    match = re.search(r'/bitstream/\d+/(\d+)/(\d+)/', parsed_url.path)
    if match:
        return f'__{match.group(1)}_{match.group(2)}'
    url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
    return f'__{url_hash}'


def build_bitstream_url_candidates(url: str, collection_base_url: str) -> list[str]:
    '''Return download URL candidates for one harvested bitstream URL.

    The goal is to try the most likely working URL first, while keeping the original harvested URL as a fallback.

    This function exists because some DSpace instances expose bitstream URLs in METS
    (Metadata Encoding and Transmission Standard:
    https://en.wikipedia.org/wiki/Metadata_Encoding_and_Transmission_Standard, the XML package format used here
    through OAI-PMH, Open Archives Initiative Protocol for Metadata Harvesting:
    https://en.wikipedia.org/wiki/Open_Archives_Initiative_Protocol_for_Metadata_Harvesting) that technically
    work, but are very slow in practice. In one observed case, the harvested original URL on an alternate
    origin/port was hanging, while the same-host collection origin URL downloaded quickly.

    Examples:
    - `url='http://repo.example.org:8080/bitstream/1234/5/1/file.pdf'` and
      `collection_base_url='https://repo.example.org'` returns
      `['https://repo.example.org/bitstream/1234/5/1/file.pdf',`
      `'http://repo.example.org:8080/bitstream/1234/5/1/file.pdf']`
    - `url='/bitstream/1234/5/1/file.pdf'` and `collection_base_url='https://repo.example.org'` returns
      `['https://repo.example.org/bitstream/1234/5/1/file.pdf']`
    - `url='https://cdn.example.org/bitstream/1234/5/1/file.pdf'` and
      `collection_base_url='https://repo.example.org'` returns only the original CDN URL, because it is not
      the same host.
    '''
    candidates: list[str] = []
    seen: set[str] = set()

    # Keep insertion order and avoid duplicates. Example: if normalization produces the same URL as the harvested
    # one, we still want only one candidate in the final list.
    def add(candidate: str) -> None:
        if candidate and candidate not in seen:
            seen.add(candidate)
            candidates.append(candidate)

    # Convert relative METS URLs into absolute URLs using the collection base. Example:
    #   base='https://repo.example.org', url='/bitstream/1234/5/1/file.pdf'
    # becomes:
    #   'https://repo.example.org/bitstream/1234/5/1/file.pdf'
    absolute_url = urllib.parse.urljoin(f'{collection_base_url.rstrip("/")}/', url)

    # Parse both URLs so we can compare hostnames and safely rebuild a normalized candidate without string slicing.
    # Example:
    #   before: `absolute_url = 'http://repo.example.org:8080/bitstream/1234/5/1/file.pdf'`
    #   after:  `parsed_absolute_url.netloc == 'repo.example.org:8080'`
    #           `parsed_absolute_url.hostname == 'repo.example.org'`
    #           `parsed_absolute_url.path == '/bitstream/1234/5/1/file.pdf'`
    parsed_absolute_url = urllib.parse.urlparse(absolute_url)

    # Example:
    #   before: `collection_base_url = 'https://repo.example.org'`
    #   after:  `parsed_collection_base_url.netloc == 'repo.example.org'`
    #           `parsed_collection_base_url.hostname == 'repo.example.org'`
    #           `parsed_collection_base_url.path == ''`
    parsed_collection_base_url = urllib.parse.urlparse(collection_base_url)

    # Detect the "same DSpace, different origin" case. Example:
    #   harvested URL:  'http://repo.example.org:8080/bitstream/...'
    #   collection URL: 'https://repo.example.org'
    # Hostname matches (`repo.example.org`), so we can try the collection origin first.
    same_host = (
        parsed_absolute_url.hostname
        and parsed_collection_base_url.hostname
        and parsed_absolute_url.hostname.casefold() == parsed_collection_base_url.hostname.casefold()
    )

    # Only normalize real bitstream paths. We do not rewrite unrelated URLs. Example:
    #   '/bitstream/1234/5/1/file.pdf' -> normalize
    #   '/themes/Mirage2/images/logo.png' -> do not normalize
    if same_host and parsed_absolute_url.path.startswith('/bitstream/'):
        # Rebuild the URL with the collection scheme + netloc, but keep the harvested bitstream path, query
        # string, and fragment. Example:
        #   absolute URL:   'http://repo.example.org:8080/bitstream/...'
        #   collection base: 'https://repo.example.org'
        # becomes:
        #   'https://repo.example.org/bitstream/...'
        normalized_url = urllib.parse.urlunparse(
            (
                parsed_collection_base_url.scheme,
                parsed_collection_base_url.netloc,
                parsed_absolute_url.path,
                parsed_absolute_url.params,
                parsed_absolute_url.query,
                parsed_absolute_url.fragment,
            )
        )
        add(normalized_url)

    # Always keep the harvested URL as a fallback, whether it was absolute or derived from a relative URL.
    add(absolute_url)
    return candidates


def parse_non_negative_int(name: str, default: int) -> int:
    raw = os.environ.get(name, str(default))
    if not re.fullmatch(r'\d+', raw):
        raise SystemExit(f'{name} must be a non-negative integer (got: {raw})')
    return int(raw)


def parse_non_negative_float(name: str, default: float) -> float:
    raw = os.environ.get(name, str(default))
    if not re.fullmatch(r'\d+(?:\.\d+)?', raw):
        raise SystemExit(f'{name} must be a non-negative number (got: {raw})')
    return float(raw)


def parse_check_commons_hash() -> bool:
    raw = os.environ.get('CHECK_COMMONS_HASH', '1')
    if raw not in {'0', '1'}:
        raise SystemExit(f'CHECK_COMMONS_HASH must be 0 or 1 (got: {raw})')
    return raw == '1'


def parse_bool_env(name: str, default: bool) -> bool:
    raw = os.environ.get(name, '1' if default else '0')
    if raw not in {'0', '1'}:
        raise SystemExit(f'{name} must be 0 or 1 (got: {raw})')
    return raw == '1'


def request_bytes(
    url: str,
    *,
    log_path: Path,
    tries: int,
    timeout: float,
    wait_retry: float,
    headers: dict[str, str] | None = None,
    method: str = 'GET',
) -> bytes:
    request_headers = {'User-Agent': HTTP_USER_AGENT}
    if headers:
        request_headers.update(headers)

    last_error: Exception | None = None
    for attempt in range(1, tries + 1):
        request = urllib.request.Request(url, headers=request_headers, method=method)
        try:
            with urllib.request.urlopen(request, timeout=timeout) as response:
                return response.read()
        except Exception as exc:  # noqa: BLE001
            last_error = exc
            log_error(log_path, f'{method} {url} failed on attempt {attempt}/{tries}: {exc}')
            if attempt == tries:
                break
            time.sleep(wait_retry)

    if last_error is None:
        raise RuntimeError(f'Failed to fetch {url}')
    raise last_error


def request_json(
    url: str,
    *,
    log_path: Path,
    tries: int,
    timeout: float,
    wait_retry: float,
    headers: dict[str, str] | None = None,
) -> dict:
    payload = request_bytes(
        url,
        log_path=log_path,
        tries=tries,
        timeout=timeout,
        wait_retry=wait_retry,
        headers=headers,
    )
    try:
        return json.loads(payload.decode('utf-8'))
    except json.JSONDecodeError as exc:
        log_error(log_path, f'Invalid JSON from {url}: {exc}')
        raise


def fetch_http_status(
    url: str,
    *,
    log_path: Path,
    timeout: float,
    method: str = 'HEAD',
) -> int | None:
    request = urllib.request.Request(url, headers={'User-Agent': HTTP_USER_AGENT}, method=method)
    try:
        with urllib.request.urlopen(request, timeout=timeout) as response:
            return getattr(response, 'status', response.getcode())
    except urllib.error.HTTPError as exc:
        return exc.code
    except Exception as exc:  # noqa: BLE001
        log_error(log_path, f'{method} {url} failed while checking status: {exc}')
        return None


def fetch_item_handle_status(base_url: str, item_handle: str, fetch_log: Path) -> int | None:
    handle_url = f'{base_url}/handle/{item_handle}'
    status_code = fetch_http_status(handle_url, log_path=fetch_log, timeout=15.0, method='HEAD')
    if status_code in {405, 501}:
        return fetch_http_status(handle_url, log_path=fetch_log, timeout=15.0, method='GET')
    return status_code


def fetch_remote_size(url: str, download_log: Path) -> int | None:
    try:
        request = urllib.request.Request(url, headers={'User-Agent': HTTP_USER_AGENT}, method='HEAD')
        with urllib.request.urlopen(request, timeout=45) as response:
            content_length = response.headers.get('Content-Length')
            if content_length and content_length.isdigit():
                return int(content_length)
    except urllib.error.HTTPError as exc:
        log_error(download_log, f'HEAD {url} failed: {exc}')
    except urllib.error.URLError as exc:
        log_error(download_log, f'HEAD {url} failed: {exc}')
    return None


def format_download_error(exc: Exception) -> str:
    if isinstance(exc, urllib.error.HTTPError):
        return f'HTTP {exc.code}'
    message = str(exc).strip()
    if not message:
        return type(exc).__name__
    if isinstance(exc, urllib.error.URLError) and getattr(exc, 'reason', None) is not None:
        reason = str(exc.reason).strip()
        if reason:
            return reason
    return message


def is_not_found_failure(failure_detail: str | None) -> bool:
    return failure_detail == 'HTTP 404'


def download_file(
    url: str,
    dst: Path,
    download_log: Path,
    *,
    collection_base_url: str,
) -> tuple[str | None, str | None]:
    harvested_absolute_url = urllib.parse.urljoin(f'{collection_base_url.rstrip("/")}/', url)
    url_candidates = build_bitstream_url_candidates(url, collection_base_url)
    local_size = dst.stat().st_size if dst.exists() else 0
    remote_size = None
    if local_size:
        for candidate_url in url_candidates:
            remote_size = fetch_remote_size(candidate_url, download_log)
            if remote_size is not None:
                break

    if remote_size is not None and local_size == remote_size:
        return 'skipped', None

    if remote_size is not None and local_size > remote_size:
        dst.unlink()
        local_size = 0

    tries = 20
    timeout = 45
    wait_retry = 2.0
    last_failure_detail = None

    for attempt in range(1, tries + 1):
        for candidate_url in url_candidates:
            resume_from = dst.stat().st_size if dst.exists() else 0
            headers = {'User-Agent': HTTP_USER_AGENT}
            mode = 'wb'
            result = 'downloaded'
            if resume_from:
                headers['Range'] = f'bytes={resume_from}-'
                mode = 'ab'
                result = 'resumed'

            request = urllib.request.Request(candidate_url, headers=headers)
            try:
                with urllib.request.urlopen(request, timeout=timeout) as response:
                    status_code = getattr(response, 'status', response.getcode())
                    if resume_from and status_code != 206:
                        mode = 'wb'
                        result = 'downloaded'
                    with dst.open(mode) as handle:
                        shutil.copyfileobj(response, handle, length=1024 * 1024)
                    return result, None
            except urllib.error.HTTPError as exc:
                last_failure_detail = format_download_error(exc)
                if exc.code == 416:
                    return 'skipped', None
                if exc.code == 404 and candidate_url != harvested_absolute_url:
                    log_error(
                        download_log,
                        f'DOWNLOAD {candidate_url} -> {dst} failed on attempt {attempt}/{tries}: {exc}; '
                        'stopping without trying the harvested fallback URL',
                    )
                    return None, last_failure_detail
                log_error(
                    download_log,
                    f'DOWNLOAD {candidate_url} -> {dst} failed on attempt {attempt}/{tries}: {exc}',
                )
            except Exception as exc:  # noqa: BLE001
                last_failure_detail = format_download_error(exc)
                log_error(
                    download_log,
                    f'DOWNLOAD {candidate_url} -> {dst} failed on attempt {attempt}/{tries}: {exc}',
                )

        if attempt < tries:
            time.sleep(wait_retry)

    return None, last_failure_detail


def commons_match_by_sha1(
    file_path: Path,
    commons_api_base: str,
    commons_user_agent: str,
    fetch_log: Path,
    commons_duplicates: Path,
) -> int:
    file_hash = sha1_file(file_path)
    query = urllib.parse.urlencode(
        {
            'action': 'query',
            'list': 'allimages',
            'format': 'json',
            'aisha1': file_hash,
        }
    )
    api_url = f'{commons_api_base}?{query}'

    try:
        data = request_json(
            api_url,
            log_path=fetch_log,
            tries=5,
            timeout=30,
            wait_retry=2.0,
            headers={'User-Agent': commons_user_agent},
        )
    except Exception:  # noqa: BLE001
        return 2

    all_images = data.get('query', {}).get('allimages')
    if not isinstance(all_images, list):
        return 2
    if not all_images:
        return 1

    titles = ';'.join(str(item.get('title', '')) for item in all_images)
    append_text(
        commons_duplicates,
        f'{file_hash}\t{file_path}\t{api_url}\t{titles}\n',
    )
    return 0


def download_with_optional_commons_check(
    url: str,
    dst: Path,
    *,
    collection_base_url: str,
    check_commons_hash: bool,
    commons_api_base: str,
    commons_user_agent: str,
    fetch_log: Path,
    download_log: Path,
    commons_duplicates: Path,
    counters: dict[str, int],
) -> tuple[str | None, str | None]:
    download_result, failure_detail = download_file(
        url,
        dst,
        download_log,
        collection_base_url=collection_base_url,
    )
    if download_result is None:
        return None, failure_detail

    if not check_commons_hash:
        return download_result, None

    rc = commons_match_by_sha1(
        dst,
        commons_api_base,
        commons_user_agent,
        fetch_log,
        commons_duplicates,
    )
    if rc == 0:
        dst.unlink(missing_ok=True)
        counters['commons_removed'] += 1
        return 'removed_on_commons', None
    elif rc == 2:
        counters['commons_check_errors'] += 1

    return download_result, None


def directory_size_bytes(path: Path) -> int:
    total = 0
    for file_path in path.rglob('*'):
        if file_path.is_file():
            total += file_path.stat().st_size
    return total


def sha1_file(path: Path) -> str:
    digest = hashlib.sha1()
    with path.open('rb') as handle:
        while True:
            chunk = handle.read(1024 * 1024)
            if not chunk:
                break
            digest.update(chunk)
    return digest.hexdigest()


def human_size(size_bytes: int) -> str:
    units = ['B', 'K', 'M', 'G', 'T']
    size = float(size_bytes)
    for unit in units:
        if size < 1024.0 or unit == units[-1]:
            if unit == 'B':
                return f'{int(size)}{unit}'
            return f'{size:.0f}{unit}'
        size /= 1024.0
    return f'{size_bytes}B'


def step3_status_label(result: str | None, failure_detail: str | None) -> str:
    if result == 'skipped':
        return 'skipped existing'
    if result == 'resumed':
        return 'resumed'
    if result == 'removed_on_commons':
        return 'removed on Commons'
    if result == 'downloaded':
        return 'downloaded'
    if is_not_found_failure(failure_detail):
        return 'not found (HTTP 404)'
    if failure_detail:
        return f'failed ({failure_detail})'
    return 'failed'


def parse_oai_xml(payload: bytes, source_url: str) -> etree.Element:
    try:
        return etree.fromstring(payload)
    except etree.ParseError as exc:
        raise OAIError(f'Invalid XML from {source_url}: {exc}') from exc


def get_oai_resumption_token(root: etree.Element) -> str:
    token = root.find('.//oai:resumptionToken', OAI_NS)
    if token is None or token.text is None:
        return ''
    return token.text.strip()


def iter_oai_pages(
    oai_base_url: str,
    *,
    verb: str,
    params: dict[str, str],
    fetch_log: Path,
    request_delay: float,
) -> Iterator[tuple[int, etree.Element]]:
    resumption_token = ''
    page_number = 0

    while True:
        page_number += 1
        if not resumption_token:
            query = {'verb': verb, **params}
        else:
            query = {'verb': verb, 'resumptionToken': resumption_token}
        url = f'{oai_base_url}?{urllib.parse.urlencode(query)}'
        payload = request_bytes(
            url,
            log_path=fetch_log,
            tries=10,
            timeout=45,
            wait_retry=2.0,
        )
        root = parse_oai_xml(payload, url)
        error = root.find('oai:error', OAI_NS)
        if error is not None:
            error_code = error.attrib.get('code', 'unknown')
            error_message = (error.text or '').strip()
            if verb == 'ListRecords' and error_code == 'noRecordsMatch':
                return
            raise OAIError(f'OAI error for {url}: {error_code}: {error_message}')

        yield page_number, root

        resumption_token = get_oai_resumption_token(root)
        if not resumption_token:
            break
        if request_delay:
            time.sleep(request_delay)


def load_set_map(oai_base_url: str, fetch_log: Path, request_delay: float) -> tuple[dict[str, str], int]:
    set_map: dict[str, str] = {}
    page_count = 0

    for page_count, root in iter_oai_pages(
        oai_base_url,
        verb='ListSets',
        params={},
        fetch_log=fetch_log,
        request_delay=request_delay,
    ):
        for set_element in root.findall('.//oai:set', OAI_NS):
            set_spec = (set_element.findtext('oai:setSpec', default='', namespaces=OAI_NS) or '').strip()
            set_name = (set_element.findtext('oai:setName', default='', namespaces=OAI_NS) or '').strip()
            if set_spec:
                set_map[set_spec] = set_name or set_spec

    return set_map, page_count


def resolve_input_set(collection_handle: str, set_map: dict[str, str]) -> tuple[str, str]:
    handle_token = collection_handle.replace('/', '_')
    candidates = [f'col_{handle_token}', f'com_{handle_token}']

    for candidate in candidates:
        if candidate in set_map:
            return candidate, set_map[candidate]

    raise SystemExit(
        f'Could not find an OAI-PMH collection/community set for handle {collection_handle}. '
        'Expected one of: '
        + ', '.join(candidates)
    )


def parse_handle_from_identifier(identifier: str) -> str:
    match = re.search(r'(\d+/\d+)$', identifier)
    if match:
        return match.group(1)
    return identifier


def extract_item_collection_dir(
    set_specs: list[str],
    *,
    input_set_spec: str,
    input_set_name: str,
    set_map: dict[str, str],
) -> str:
    if input_set_spec.startswith('col_'):
        return ''

    for set_spec in set_specs:
        if set_spec.startswith('col_'):
            set_name = set_map.get(set_spec, set_spec)
            safe = sanitize_collection_dir_name(set_name)
            if safe:
                return safe

    safe = sanitize_collection_dir_name(input_set_name)
    return safe or 'unknown_collection'


def extract_original_bitstream_rows(record: etree.Element) -> list[tuple[str, int | None]]:
    rows: list[tuple[str, int | None]] = []
    for file_group in record.findall('.//mets:fileGrp', OAI_NS):
        if file_group.get('USE') != 'ORIGINAL':
            continue
        for file_element in file_group.findall('mets:file', OAI_NS):
            size_bytes = None
            size_attr = file_element.get('SIZE')
            if size_attr and size_attr.isdigit():
                size_bytes = int(size_attr)
            flocat = file_element.find('mets:FLocat', OAI_NS)
            if flocat is None:
                continue
            href = (flocat.attrib.get(XLINK_HREF) or '').strip()
            if href:
                rows.append((href, size_bytes))
    return sorted(set(rows))


def plan_download_rows(
    download_rows: list[tuple[str, str, int | None]],
) -> tuple[list[tuple[str, str, int | None]], int]:
    base_rows: list[tuple[str, str, int | None, str, str]] = []
    base_key_counts: Counter[tuple[str, str, str]] = Counter()

    for index, (item_collection_dir, url, size_bytes) in enumerate(download_rows, start=1):
        file_name = extract_file_name_from_url(url, f'download_{index}')
        year_folder = year_folder_from_filename(file_name)
        base_rows.append((item_collection_dir, url, size_bytes, year_folder, file_name))
        base_key_counts[(item_collection_dir, year_folder, file_name)] += 1

    collision_handled_count = 0
    first_pass_rows: list[tuple[tuple[str, str, str], str, int | None]] = []
    first_pass_key_counts: Counter[tuple[str, str, str]] = Counter()

    for item_collection_dir, url, size_bytes, year_folder, file_name in base_rows:
        final_file_name = file_name
        if base_key_counts[(item_collection_dir, year_folder, file_name)] > 1:
            final_file_name = append_suffix_to_file_name(file_name, bitstream_identity_suffix(url))
            collision_handled_count += 1
        final_key = (item_collection_dir, year_folder, final_file_name)
        first_pass_rows.append((final_key, url, size_bytes))
        first_pass_key_counts[final_key] += 1

    planned_rows: list[tuple[str, str, int | None]] = []
    for (item_collection_dir, year_folder, final_file_name), url, size_bytes in first_pass_rows:
        resolved_file_name = final_file_name
        if first_pass_key_counts[(item_collection_dir, year_folder, final_file_name)] > 1:
            url_hash = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
            resolved_file_name = append_suffix_to_file_name(final_file_name, f'__{url_hash}')
        relative_path = str(Path(item_collection_dir) / year_folder / resolved_file_name)
        planned_rows.append((relative_path, url, size_bytes))

    planned_rows.sort(key=lambda row: natural_relative_path_key(row[0]))
    return planned_rows, collision_handled_count


def harvest_download_rows(
    base_url: str,
    oai_base_url: str,
    *,
    input_set_spec: str,
    input_set_name: str,
    set_map: dict[str, str],
    validate_item_handles: bool,
    request_delay: float,
    fetch_log: Path,
    run_log: Path,
    download_urls_raw: Path,
    download_urls: Path,
    invalid_oai_items: Path,
    items_without_files: Path,
) -> tuple[list[tuple[str, str, int | None]], int, int, int, int, int]:
    raw_rows: list[tuple[str, str, int | None]] = []
    handle_status_cache: dict[str, int | None] = {}
    item_count = 0
    invalid_item_count = 0
    nofile_count = 0
    page_count = 0

    download_urls_raw.write_text('', encoding='utf-8')
    invalid_oai_items.write_text('', encoding='utf-8')
    items_without_files.write_text('', encoding='utf-8')

    for page_count, root in iter_oai_pages(
        oai_base_url,
        verb='ListRecords',
        params={
            'metadataPrefix': 'mets',
            'set': input_set_spec,
        },
        fetch_log=fetch_log,
        request_delay=request_delay,
    ):
        for record in root.findall('.//oai:record', OAI_NS):
            header = record.find('oai:header', OAI_NS)
            if header is None or header.attrib.get('status') == 'deleted':
                continue

            identifier = (header.findtext('oai:identifier', default='', namespaces=OAI_NS) or '').strip()
            item_handle = parse_handle_from_identifier(identifier)
            if validate_item_handles:
                status_code = handle_status_cache.get(item_handle)
                if status_code is None and item_handle not in handle_status_cache:
                    status_code = fetch_item_handle_status(base_url, item_handle, fetch_log)
                    handle_status_cache[item_handle] = status_code
                if status_code == 404:
                    invalid_item_count += 1
                    append_text(invalid_oai_items, f'{item_handle}\t404\t{identifier}\n')
                    continue
            set_specs = [
                (element.text or '').strip()
                for element in header.findall('oai:setSpec', OAI_NS)
                if (element.text or '').strip()
            ]
            item_collection_dir = extract_item_collection_dir(
                set_specs,
                input_set_spec=input_set_spec,
                input_set_name=input_set_name,
                set_map=set_map,
            )
            bitstream_rows = extract_original_bitstream_rows(record)

            item_count += 1
            if not bitstream_rows:
                nofile_count += 1
                append_text(items_without_files, f'{parse_handle_from_identifier(identifier)}\n')
                continue

            for url, size_bytes in bitstream_rows:
                row = (item_collection_dir, url, size_bytes)
                raw_rows.append(row)
                size_field = '' if row[2] is None else str(row[2])
                append_text(download_urls_raw, f'{row[0]}\t{row[1]}\t{size_field}\n')

            if item_count % 500 == 0:
                log(f'  OAI records processed: {item_count}', run_log)

    deduped_rows = sorted(set(raw_rows))
    planned_rows, collision_handled_count = plan_download_rows(deduped_rows)
    with download_urls.open('w', encoding='utf-8') as handle:
        for relative_path, url, size_bytes in planned_rows:
            size_field = '' if size_bytes is None else str(size_bytes)
            handle.write(f'{relative_path}\t{url}\t{size_field}\n')

    return planned_rows, item_count, invalid_item_count, nofile_count, page_count, collision_handled_count


def run(argv: list[str] | None = None) -> int:
    global ACTIVE_RUN_LOG
    parser = argparse.ArgumentParser(
        prog='dspace_wgetter.py',
        description='Download DSpace files through OAI-PMH.',
    )
    parser.add_argument('collection_handle_url')
    parser.add_argument('output_dir', nargs='?')
    args = parser.parse_args(argv)

    base_url, handle_path = parse_collection_url(args.collection_handle_url)
    collection_handle = handle_path.removeprefix('/handle/')
    collection_id = collection_handle.split('/')[-1]
    oai_base_url = f'{base_url}/oai/request'

    request_delay = parse_non_negative_float('REQUEST_DELAY', 0.0)
    download_delay = parse_non_negative_float('DOWNLOAD_DELAY', 0.0)
    check_commons_hash = parse_check_commons_hash()
    resume_failed_downloads = parse_bool_env('RESUME_FAILED_DOWNLOADS', True)
    validate_item_handles = parse_bool_env('VALIDATE_ITEM_HANDLES', True)
    failed_retry_passes = parse_non_negative_int('FAILED_RETRY_PASSES', 3)
    failed_retry_delay = parse_non_negative_float('FAILED_RETRY_DELAY', 60.0)
    commons_user_agent = os.environ.get('COMMONS_USER_AGENT', 'dspace_wgetter (Commons SHA1 check)')
    commons_api_base = os.environ.get('COMMONS_API_BASE', 'https://commons.wikimedia.org/w/api.php')

    set_map, set_pages = load_set_map(oai_base_url, Path('/dev/null'), request_delay)
    input_set_spec, input_set_name = resolve_input_set(collection_handle, set_map)
    root_dir_name = sanitize_collection_dir_name(input_set_name) or f'download_{collection_id}'

    out_dir = Path(args.output_dir) if args.output_dir else Path.cwd() / root_dir_name
    files_dir = out_dir / 'files'
    manifest_dir = out_dir / '_manifests'
    log_dir = out_dir / '_logs'
    files_dir.mkdir(parents=True, exist_ok=True)
    manifest_dir.mkdir(parents=True, exist_ok=True)
    log_dir.mkdir(parents=True, exist_ok=True)

    run_log = log_dir / 'run.log'
    fetch_log = log_dir / 'fetch.log'
    download_log = log_dir / 'download.log'
    commons_duplicates = manifest_dir / 'commons_duplicates.tsv'
    download_urls_raw = manifest_dir / 'download_urls.raw.tsv'
    download_urls = manifest_dir / 'download_urls.tsv'
    failed_downloads = manifest_dir / 'failed_downloads.tsv'
    invalid_oai_items = manifest_dir / 'invalid_oai_items.tsv'
    items_without_files = manifest_dir / 'items_without_files.txt'
    not_found_downloads = manifest_dir / 'not_found_downloads.tsv'

    run_log.write_text('', encoding='utf-8')
    fetch_log.write_text('', encoding='utf-8')
    download_log.write_text('', encoding='utf-8')
    commons_duplicates.write_text('', encoding='utf-8')
    write_download_rows(not_found_downloads, [])
    ACTIVE_RUN_LOG = run_log

    log(f'Collection URL: {args.collection_handle_url}', run_log)
    log(f'OAI base URL: {oai_base_url}', run_log)
    log(f'OAI set spec: {input_set_spec}', run_log)
    log(f'Collection name: {input_set_name}', run_log)
    log(f'Output dir: {out_dir}', run_log)
    log(f'Files dir: {files_dir}', run_log)
    log(f'OAI set pages loaded: {set_pages}', run_log)
    if input_set_spec.startswith('col_'):
        log('Download layout: by year (files/<year>/<filename>)', run_log)
    else:
        log('Download layout: by collection and year (files/<collection>/<year>/<filename>)', run_log)
    log(
        'Bitstream URL handling: prefer the collection base origin for same-host /bitstream/ URLs, '
        'then fall back to the harvested URL',
        run_log,
    )
    log(f'Auto-resume from failed_downloads.tsv: {"enabled" if resume_failed_downloads else "disabled"}', run_log)
    log('Resume queue updates: after every processed file', run_log)
    log('HTTP 404 handling: terminal, not retried', run_log)
    log(f'Item handle validation during harvest: {"enabled" if validate_item_handles else "disabled"}', run_log)
    log(f'Retry passes for failed downloads: {failed_retry_passes}', run_log)
    log(f'Delay before retry pass: {failed_retry_delay}s', run_log)

    if check_commons_hash:
        log('Commons SHA1 duplicate check: enabled', run_log)
    else:
        log('Commons SHA1 duplicate check: disabled', run_log)

    log('Step 1/3: Resolving OAI-PMH set metadata...', run_log)
    log(f'  input handle resolved to {input_set_spec}', run_log)

    log('Step 2/3: Harvesting item records and file URLs via OAI-PMH...', run_log)
    (
        download_rows,
        item_count,
        invalid_item_count,
        nofile_count,
        record_pages,
        collision_handled_count,
    ) = harvest_download_rows(
        base_url,
        oai_base_url,
        input_set_spec=input_set_spec,
        input_set_name=input_set_name,
        set_map=set_map,
        validate_item_handles=validate_item_handles,
        request_delay=request_delay,
        fetch_log=fetch_log,
        run_log=run_log,
        download_urls_raw=download_urls_raw,
        download_urls=download_urls,
        invalid_oai_items=invalid_oai_items,
        items_without_files=items_without_files,
    )
    url_count = len(download_rows)
    active_download_rows = download_rows
    resume_rows_used = False
    saved_failed_rows_count = 0
    dropped_saved_rows_count = 0
    if resume_failed_downloads and failed_downloads.exists():
        saved_failed_rows = read_download_rows(failed_downloads)
        saved_failed_rows_count = len(saved_failed_rows)
        if saved_failed_rows:
            planned_row_set = set(download_rows)
            filtered_saved_failed_rows = [row for row in saved_failed_rows if row in planned_row_set]
            dropped_saved_rows_count = saved_failed_rows_count - len(filtered_saved_failed_rows)
            if filtered_saved_failed_rows or dropped_saved_rows_count:
                active_download_rows = filtered_saved_failed_rows
                resume_rows_used = True
    log(f'OAI record pages harvested: {record_pages}', run_log)
    log(f'Items harvested: {item_count}', run_log)
    log(f'Invalid OAI items filtered: {invalid_item_count}', run_log)
    log(f'File URLs resolved: {url_count}', run_log)
    log(f'Filename collisions handled: {collision_handled_count}', run_log)
    log(f'Items without files: {nofile_count}', run_log)
    if dropped_saved_rows_count:
        log(
            f'Stale rows dropped from failed_downloads.tsv: {dropped_saved_rows_count}',
            run_log,
        )
    if resume_rows_used:
        log(
            f'Resume queue loaded from failed_downloads.tsv: {len(active_download_rows)} remaining downloads.',
            run_log,
        )
    elif saved_failed_rows_count:
        log(
            f'Saved failed_downloads.tsv found with {saved_failed_rows_count} rows, but resume was not used.',
            run_log,
        )

    log('Step 3/3: Downloading files sequentially...', run_log)
    write_download_rows(failed_downloads, active_download_rows)

    counters = {
        'commons_check_errors': 0,
        'commons_removed': 0,
        'fail': 0,
        'ok': 0,
    }
    interrupted = False
    failed_rows: list[tuple[str, str, int | None]] = []
    not_found_rows: list[tuple[str, str, int | None]] = []
    last_index = 0
    current_row_processed = False

    try:
        active_url_count = len(active_download_rows)
        for index, (relative_path, url, size_bytes) in enumerate(active_download_rows, start=1):
            last_index = index
            current_row_processed = False
            dst = files_dir / relative_path
            dst.parent.mkdir(parents=True, exist_ok=True)
            result, failure_detail = download_with_optional_commons_check(
                url,
                dst,
                collection_base_url=base_url,
                check_commons_hash=check_commons_hash,
                commons_api_base=commons_api_base,
                commons_user_agent=commons_user_agent,
                fetch_log=fetch_log,
                download_log=download_log,
                commons_duplicates=commons_duplicates,
                counters=counters,
            )
            log(
                f'  {step3_status_label(result, failure_detail)}: {index}/{active_url_count} {dst} '
                f'(size={format_size_mb(size_bytes)})',
                run_log,
            )

            if result is not None:
                counters['ok'] += 1
            elif is_not_found_failure(failure_detail):
                not_found_rows.append((relative_path, url, size_bytes))
            else:
                failed_rows.append((relative_path, url, size_bytes))
            write_download_rows(not_found_downloads, not_found_rows)
            write_download_rows(
                failed_downloads,
                remaining_resume_rows(failed_rows, active_download_rows, index),
            )
            current_row_processed = True

            if index % 500 == 0:
                size_now = human_size(directory_size_bytes(files_dir))
                log(
                    f'  downloaded: {index} / {active_url_count} '
                    f'(ok={counters["ok"]} fail={len(failed_rows)} '
                    f'removed={counters["commons_removed"]} '
                    f'commons_check_errors={counters["commons_check_errors"]} '
                    f'size={size_now})',
                    run_log,
                )
            if download_delay:
                time.sleep(download_delay)
    except KeyboardInterrupt:
        interrupted = True
        log('Interrupted (Ctrl+C). Saving remaining downloads for resume...', run_log)
    processed_count = last_index if current_row_processed else max(last_index - 1, 0)
    write_download_rows(
        failed_downloads,
        remaining_resume_rows(failed_rows, active_download_rows, processed_count),
    )
    with failed_downloads.open(encoding='utf-8') as handle:
        counters['fail'] = sum(1 for line in handle if line.strip())

    if failed_retry_passes > 0 and not interrupted:
        for retry_pass in range(1, failed_retry_passes + 1):
            remaining_rows = read_download_rows(failed_downloads)

            if not remaining_rows:
                break

            try:
                if failed_retry_delay:
                    log(
                        f'Retry pass {retry_pass}/{failed_retry_passes}: waiting {failed_retry_delay}s '
                        f'before retrying {len(remaining_rows)} failed downloads...',
                        run_log,
                    )
                    time.sleep(failed_retry_delay)
                else:
                    log(
                        f'Retry pass {retry_pass}/{failed_retry_passes}: '
                        f'retrying {len(remaining_rows)} failed downloads...',
                        run_log,
                    )
            except KeyboardInterrupt:
                interrupted = True
                log('Interrupted (Ctrl+C). Saving remaining downloads for resume...', run_log)
                break

            recovered = 0
            retry_failed_rows: list[tuple[str, str, int | None]] = []
            last_retry_index = 0
            current_retry_row_processed = False

            try:
                for retry_index, (relative_path, url, size_bytes) in enumerate(remaining_rows, start=1):
                    last_retry_index = retry_index
                    current_retry_row_processed = False
                    dst = files_dir / relative_path
                    dst.parent.mkdir(parents=True, exist_ok=True)
                    result, failure_detail = download_with_optional_commons_check(
                        url,
                        dst,
                        collection_base_url=base_url,
                        check_commons_hash=check_commons_hash,
                        commons_api_base=commons_api_base,
                        commons_user_agent=commons_user_agent,
                        fetch_log=fetch_log,
                        download_log=download_log,
                        commons_duplicates=commons_duplicates,
                        counters=counters,
                    )
                    log(
                        f'  retry {step3_status_label(result, failure_detail)}: '
                        f'{retry_index}/{len(remaining_rows)} {dst} '
                        f'(size={format_size_mb(size_bytes)})',
                        run_log,
                    )

                    if result is not None:
                        recovered += 1
                    elif is_not_found_failure(failure_detail):
                        not_found_rows.append((relative_path, url, size_bytes))
                    else:
                        retry_failed_rows.append((relative_path, url, size_bytes))
                    write_download_rows(not_found_downloads, not_found_rows)
                    write_download_rows(
                        failed_downloads,
                        remaining_resume_rows(retry_failed_rows, remaining_rows, retry_index),
                    )
                    current_retry_row_processed = True

                    if download_delay:
                        time.sleep(download_delay)
            except KeyboardInterrupt:
                interrupted = True
                log('Interrupted (Ctrl+C). Saving remaining downloads for resume...', run_log)
            retry_processed_count = last_retry_index if current_retry_row_processed else max(last_retry_index - 1, 0)
            write_download_rows(
                failed_downloads,
                remaining_resume_rows(retry_failed_rows, remaining_rows, retry_processed_count),
            )
            counters['ok'] += recovered
            with failed_downloads.open(encoding='utf-8') as handle:
                counters['fail'] = sum(1 for _ in handle if _.strip())
            log(
                f'  retry pass {retry_pass} done: recovered={recovered} remaining_failed={counters["fail"]}',
                run_log,
            )
            if interrupted:
                break

    final_size = human_size(directory_size_bytes(files_dir))
    log('Done.', run_log)
    log('Summary:', run_log)
    log(f'  Collection handle:   {collection_handle}', run_log)
    log(f'  OAI set spec:        {input_set_spec}', run_log)
    log(f'  Items harvested:     {item_count}', run_log)
    log(f'  Invalid OAI items:   {invalid_item_count}', run_log)
    log(f'  File URLs resolved:  {url_count}', run_log)
    log(f'  Download queue used: {len(active_download_rows)}', run_log)
    log(f'  Resumed from saved queue: {"yes" if resume_rows_used else "no"}', run_log)
    log(f'  Items without files: {nofile_count}', run_log)
    log(f'  Download ok:         {counters["ok"]}', run_log)
    log(f'  Download failed:     {counters["fail"]}', run_log)
    log(f'  Not found (HTTP 404): {len(not_found_rows)}', run_log)
    log(f'  Interrupted:         {"yes" if interrupted else "no"}', run_log)
    log(f'  Removed (on Commons): {counters["commons_removed"]}', run_log)
    log(f'  Commons check errors: {counters["commons_check_errors"]}', run_log)
    log(f'  Files size:          {final_size}', run_log)
    log(f'  Output files dir:    {files_dir}', run_log)
    log(f'  Manifests dir:       {manifest_dir}', run_log)
    log(f'  Logs dir:            {log_dir}', run_log)
    log(f'  Commons duplicates:  {commons_duplicates}', run_log)
    log(f'  Invalid OAI items:   {invalid_oai_items}', run_log)
    log(f'  Not found manifest:  {not_found_downloads}', run_log)
    return 130 if interrupted else 0


def main() -> None:
    signal.signal(signal.SIGINT, handle_sigint)
    try:
        raise SystemExit(run())
    except KeyboardInterrupt:
        message = 'Interrupted (Ctrl+C).'
        if ACTIVE_RUN_LOG is not None:
            log(message, ACTIVE_RUN_LOG)
        else:
            print(message, flush=True)
        raise SystemExit(130)


if __name__ == '__main__':
    main()
