# SPDX-FileCopyrightText: 2016 Curtis L. Olson
# SPDX-License-Identifier: GPL-2.0-or-later

"""Generate and update FlightGear aircraft catalogs."""

import argparse
import datetime
import hashlib                  # md5
import lxml.etree as ET
import os
import re
import shutil
import subprocess
import sys

from flightgear.meta.aircraft_catalogs import catalogTags
from flightgear.meta.aircraft_catalogs import catalog
from flightgear.meta.aircraft_catalogs.catalog import make_aircraft_node, \
    make_aircraft_zip, parse_config_file, parse_template_file

PROGNAME = os.path.basename(sys.argv[0])
CATALOG_VERSION = 4

parser = argparse.ArgumentParser()
parser.add_argument("--update", help="Update/pull SCM source",
                    action="store_true")
parser.add_argument("--no-update",
                    help="Disable updating from SCM source",
                    action="store_true")
parser.add_argument("--clean", help="Force regeneration of all zip files",
                    action="store_true")
parser.add_argument("--quiet", help="Only print warnings and errors",
                    action="store_true")

parser.add_argument('-B', "--build", help="Specify the build/download directory",
                    action="store", default=os.getcwd())

parser.add_argument("dir", help="Catalog directory")
args = parser.parse_args()

# We avoid changing the current directory but will feel safer this way. :)
args.build = os.path.abspath(args.build)
args.dir = os.path.abspath(args.dir)

includes = []
mirrors = [] # mirror base URLs

catalog.quiet = args.quiet

def buildPath(path):
    return os.path.join(args.build, path)

def sourcePath(path):
    return os.path.join(args.dir, path)

# xml node (robust) get text helper
def get_xml_text(e):
    if e is not None and e.text is not None:
        return e.text
    else:
        return ''

def get_xml_bool(e):
    s = get_xml_text(e)
    return s in ('yes', 'true', '1')


# use svn commands to report the last change date within dir
def last_change_date_svn(dir_):
    env = os.environ.copy()
    env["LC_ALL"] = "C.UTF-8"
    command = ['svn', 'info', dir_]
    output = subprocess.run(command, check=True, env=env,
                            stdout=subprocess.PIPE, encoding="utf-8").stdout

    mo = re.search(r'^Last Changed Date: (\d+)-(\d+)-(\d+)', output,
                   re.MULTILINE)
    if mo:
        rev_str = mo.group(1) + mo.group(2) + mo.group(3)
        return int(rev_str)
    else:
        print(f"{PROGNAME}: unable to find the last change date of directory "
              f"{dir_!r} in Subversion repository. Output of 'svn info' was:"
              f"\n\n{output}")
        sys.exit(1)


# find the most recent mtime within a directory subtree
def scan_dir_for_change_date_mtime(path):
    maxsec = 0
    names = os.listdir(path)
    for name in names:
        fullname = os.path.join(path, name)
        if name == '.' or name == '..':
            pass
        elif os.path.isdir( fullname ):
            mtime = scan_dir_for_change_date_mtime( fullname )
            if mtime > maxsec:
                maxsec = mtime
        else:
            mtime = os.path.getmtime( fullname )
            if mtime > maxsec:
                maxsec = mtime
    return maxsec


def get_md5sum(file):
    with open(file, 'rb') as f:
        md5sum = hashlib.md5(f.read()).hexdigest()

    return md5sum


def copy_previews_for_variant(variant, package_name, package_dir, previews_dir):
    if not 'previews' in variant:
        return

    for preview in variant['previews']:
        preview_src = os.path.join(package_dir, preview['path'])
        preview_dst = os.path.join(previews_dir, package_name + '_' + preview['path'])
        #print(preview_src, preview_dst, preview['path'])
        dir = os.path.dirname(preview_dst)
        if not os.path.isdir(dir):
            os.makedirs(dir)
        if os.path.exists(preview_src):
            shutil.copy2(preview_src, preview_dst)


def copy_previews_for_package(package, variants, package_name, package_dir, previews_dir):
    copy_previews_for_variant(package, package_name, package_dir, previews_dir)
    for v in variants:
        copy_previews_for_variant(v, package_name, package_dir, previews_dir)


def copy_thumbnail_for_variant(variant, package_name, package_dir, thumbnails_dir):
    if not 'thumbnail' in variant:
        return

    thumb_src = os.path.join(package_dir, variant['thumbnail'])
    thumb_dst = os.path.join(thumbnails_dir, package_name + '_' + variant['thumbnail'])

    dir = os.path.dirname(thumb_dst)
    if not os.path.isdir(dir):
        os.makedirs(dir)
    if os.path.exists(thumb_src):
        shutil.copy2(thumb_src, thumb_dst)


def copy_thumbnails_for_package(package, variants, package_name, package_dir, thumbnails_dir):
    copy_thumbnail_for_variant(package, package_name, package_dir, thumbnails_dir)

    # and now each variant in turn
    for v in variants:
        copy_thumbnail_for_variant(v, package_name, package_dir, thumbnails_dir)


def process_aircraft_dir(name, repo_path, repo_type):
    global includes
    global download_base
    global output_dir
    global valid_zips
    global previews_dir
    global mirrors

    aircraft_dir = os.path.join(repo_path, name)
    if not os.path.isdir(aircraft_dir):
        return

    (package, variants) = catalog.scan_aircraft_dir(aircraft_dir, includes)
    if package is None:
        print("skipping: %s (no -set.xml files)" % name)
        return

    if not args.quiet:
        print("%s:" % name)

    package_node = make_aircraft_node(name, package, variants, download_base, mirrors)

    download_url = download_base + name + '.zip'
    if 'thumbnail' in package:
        # this is never even used, but breaks the script by assuming
        # all aircraft packages have thumbnails defined?
        thumbnail_url = download_base + 'thumbnails/' + name + '_' + package['thumbnail']

    # get cached md5sum if it exists
    md5sum = get_xml_text(md5sum_root.find(str('aircraft_' + name)))

    # now do the packaging and rev number stuff
    dir_mtime = scan_dir_for_change_date_mtime(aircraft_dir)
    if repo_type == 'svn':
        rev = last_change_date_svn(aircraft_dir)
    else:
        d = datetime.datetime.fromtimestamp(dir_mtime, datetime.UTC)
        rev = d.strftime("%Y%m%d")
    package_node.append( catalog.make_xml_leaf('revision', rev) )
    #print("rev: %s" % rev)
    #print("dir mtime: %s" % dir_mtime)
    zipfile = os.path.join( output_dir, name + '.zip' )
    valid_zips.append(name + '.zip')
    if not os.path.exists(zipfile) \
       or dir_mtime > os.path.getmtime(zipfile) \
       or args.clean:
        # rebuild zip file
        if not args.quiet:
            print("updating: %s" % zipfile)
        make_aircraft_zip(repo_path, name, zipfile, zip_excludes, verbose=not args.quiet)
        md5sum = get_md5sum(zipfile)
    else:
        if not args.quiet:
            print("(no change)")
        if md5sum == "":
            md5sum = get_md5sum(zipfile)
    filesize = os.path.getsize(zipfile)
    package_node.append( catalog.make_xml_leaf('md5', md5sum) )
    package_node.append( catalog.make_xml_leaf('file-size-bytes', filesize) )

    # handle md5sum cache
    node = md5sum_root.find('aircraft_' + name)
    if node is not None:
        node.text = md5sum
    else:
        md5sum_root.append( catalog.make_xml_leaf('aircraft_' + name, md5sum) )

    # handle sharing
    if share_md5sum_root is not None:
        sharedNode = share_md5sum_root.find(str('aircraft_' + name))
        if node is not None:
            shared_md5 = get_xml_text(sharedNode)
            if shared_md5 == md5sum:
                if not args.quiet:
                    print(f"Sharing zip with share catalog for: {name!r}")
                os.remove(zipfile)
                os.symlink(os.path.join( share_output_dir, name + '.zip' ), zipfile)


    # handle thumbnails
    copy_thumbnails_for_package(package, variants, name, aircraft_dir, thumbnail_dir)

    catalog_node.append(package_node)

    # copy previews for the package and variants into the
    # output directory
    copy_previews_for_package(package, variants, name, aircraft_dir, previews_dir)

def update_svn(scm):
    # check if we need to checkout rather than update
    repoPathNode = scm.find('repo-path')
    repo_path = get_xml_text(repoPathNode) if repoPathNode else get_xml_text(scm.find('path'))
    fullRepoPath = buildPath(repo_path)

    if not os.path.isdir(fullRepoPath):
        repo = get_xml_text(scm.find('repo'))
        print(f'No existing SVN repo, will checkout {repo!r}')
        subprocess.run(['svn', 'checkout', repo, fullRepoPath], check=True)
        print('Did finish SVN checkout')

    for attempts in range(5):
        r = subprocess.run(['svn', 'update', fullRepoPath])
        if r.returncode == 0:
            break

        print(f'SVN update failed for {repo_path!r}, doing cleanup')
        r = subprocess.run(['svn', 'cleanup', fullRepoPath])
        # cleanup worked, do another attempt
        if r.returncode == 0:
            continue

        # https://stackoverflow.com/questions/158664/what-should-i-do-when-svn-cleanup-fails
        # explains what we're doing here :(
        wcDbPath = os.path.join(fullRepoPath, '.svn', 'wc.db')
        print(f'SVN cleanup failed, trying manual clear of work-queue at {wcDbPath!r}')
        subprocess.run(['sqlite3', wcDbPath, 'delete from work_queue'], check=True)

def update_git(scm):
    repoPathNode = scm.find('repo-path')
    repo_path = get_xml_text(repoPathNode) if repoPathNode else get_xml_text(scm.find('path'))
    fullRepoPath = buildPath(repo_path)

    if not os.path.isdir(fullRepoPath):
        repo = get_xml_text(scm.find('repo'))
        print(f'No existing Git repo, will clone {repo!r}')
        subprocess.run(['git', 'clone', repo, fullRepoPath], check=True)
        print('Did finish Git clone')
    else:
        subprocess.run(['git', 'pull'], cwd=fullRepoPath, check=True)

def update_git_worktree(scm):
    repoPathNode = scm.find('repo-path')
    repo_path = get_xml_text(repoPathNode) if repoPathNode else get_xml_text(scm.find('path'))
    fullRepoPath = buildPath(repo_path)

    if not os.path.isdir(fullRepoPath):
        repo = get_xml_text(scm.find('repo'))
        branch = get_xml_text(scm.find('branch'))
        if not os.path.isdir(repo):
            print(f"Git worktree SCM: repo path not found: {repo!r}")
            sys.exit(1)

        print("Git worktree: creating worktree for {!r}".format(branch))
        subprocess.run(['git', 'worktree', 'add', fullRepoPath, branch],
                        cwd=sourcePath(repo), check=True)
    else:
        subprocess.run(['git', 'pull'], cwd=fullRepoPath, check=True)

def update_scm(scm):
    repo_type = get_xml_text(scm.find('type'))
    repo_path = get_xml_text(scm.find('path'))

    # XML mandated skip, with command line override.
    if not args.update:
        sn = scm.find('update')
        # get_xml_bool treats missing node as false, but we want to default to true,
        # so manually check for None here
        if sn is not None and not get_xml_bool(sn):
            return

    if repo_type == 'svn':
        update_svn(scm)
    elif repo_type == 'git':
        update_git(scm)
    elif repo_type == 'git-worktree':
        update_git_worktree(scm)
    elif repo_type == 'no-scm':
        print("No update of unmannaged files: %s" % repo_path)
    else:
        print("Unknown scm type: %s %s" % (scm, repo_path))

def update_all_scms(scm_lit):
    if args.no_update:
        print('Skipping repository updates.')
        return

    scm_list = config_node.findall('scm')
    for scm in scm_list:
        update_scm(scm)

def scan_scm(scm):
    # because of, e.g., FGData, we have 'include only' SCM providers
    # which don't need to be scanned, there are only used for XML includes
    if get_xml_bool(scm.find('include-only')):
        return

    repo_type = get_xml_text(scm.find('type'))
    repo_path = get_xml_text(scm.find('path'))
    pathOnDisk = buildPath(repo_path)

    skip_nodes = scm.findall('skip')
    skip_list = [ get_xml_text(s) for s in skip_nodes ]

    # Selective list of craft to include, overriding the skip list.
    include_nodes = scm.findall('include')
    include_list = [ get_xml_text(node) for node in include_nodes ]
    if include_list:
        skip_list.clear()

    print("Skip list: %s" % skip_list)
    print("Include list: %s" % include_list)

    names = os.listdir(pathOnDisk)
    for name in sorted(names, key=lambda s: s.lower()):
        if name in skip_list or (include_list and name not in include_list):
            if not args.quiet:
                print("Skipping: %s" % name)
            continue

        process_aircraft_dir(name, pathOnDisk, repo_type)


def upload(u, outputDir):
    if not get_xml_bool(u.find('enabled')):
        return

    upload_type = get_xml_text(u.find('type'))
    remote = get_xml_text(u.find('remote'))
    identityFile = buildPath(get_xml_text(u.find('identity')))
    if not os.path.exists(identityFile):
        print(f'Missing identity file {identityFile!r}, skipping upload')
        return

    if (upload_type == 'rsync-ssh'):
        subprocess.run(['rsync', '-avze', f'ssh -i {identityFile}', outputDir, remote])

# def get_file_stats(file):
#     with open(file, 'r') as f:
#         md5 = hashlib.md5(f.read()).hexdigest()
#     file_size = os.path.getsize(file)
#     return (md5, file_size)


if not os.path.isdir(args.dir):
    print("A valid catalog directory must be provided")
    sys.exit(1)

parser = ET.XMLParser(remove_blank_text=True)
config_node = parse_config_file(parser=parser, file_name=os.path.join(args.dir, 'catalog.config.xml'))
template_node = parse_template_file(parser=parser, file_name=os.path.join(args.dir, 'template.xml'))

md5CacheNode = config_node.find('md5-cache')
if md5CacheNode is not None:
    md5sum_file = buildPath(get_xml_text(md5CacheNode))
    if os.path.exists(md5sum_file):
        md5sum_tree = ET.parse(md5sum_file, parser)
        md5sum_root = md5sum_tree.getroot()
    else:
        md5sum_root = ET.Element('PropertyList')
        md5sum_tree = ET.ElementTree(md5sum_root)
else:
    print("Missing 'md5-cache' element in catalog config (this should specify "
          "a path relative to the build directory)")
    sys.exit(1)

# share .zip files with other output dirs
share_output_dir = get_xml_text(config_node.find('share-output'))
share_md5_file = get_xml_text(config_node.find('share-md5-sums'))
if share_output_dir != '' and share_md5_file != '':
    print("Output shared with: %s" % share_output_dir)
    share_md5sum_tree = ET.parse(share_md5_file, parser)
    share_md5sum_root = share_md5sum_tree.getroot()
else:
    share_md5sum_root = None

# SCM providers
scm_list = config_node.findall('scm')
upload_node = config_node.find('upload')

download_base = None
for i in config_node.findall("download-url"):
    url = get_xml_text(i)
    if not url.endswith('/'):
        url += '/'

    if download_base is None:
        # download_base is the first entry
        download_base = url
    else:
        mirrors.append(url)

output_dir = get_xml_text(config_node.find('local-output')) or 'output'
# if relative, it's relative to our build dir
output_dir = buildPath(output_dir)

os.makedirs(output_dir, exist_ok=True)

thumbnail_dir = os.path.join(output_dir, 'thumbnails')
if not os.path.isdir(thumbnail_dir):
    os.mkdir(thumbnail_dir)

previews_dir = os.path.join(output_dir, 'previews')
os.makedirs(previews_dir, exist_ok=True)

zip_excludes = sourcePath('zip-excludes.lst')

for i in config_node.findall("include-dir"):
    path = sourcePath(get_xml_text(i))
    if not os.path.exists(path):
        print("Missing include path: %s" % path)
        sys.exit(1)

    includes.append(path)

update_all_scms(scm_list)

# names of zip files we want (so we can identify/remove orphans)
valid_zips = []

# create the catalog tree
catalog_node = ET.Element('PropertyList')
catalog_root = ET.ElementTree(catalog_node)

# include the template configuration
for child in template_node:
    catalog_node.append(child)

# find all include SCMs
for scm in scm_list:
    # typically used for FGData, etc
    include_only = get_xml_bool(scm.find('include-only'))
    if include_only:
        repo_path = get_xml_text(scm.find('path'))
        includes.append(buildPath(repo_path))

if not args.quiet:
    print("Include paths:")
    for i in includes:
        print("\t{}".format(i))


# scan repositories for catalog information
for scm in scm_list:
    scan_scm(scm)

# write out the master catalog file
cat_file = os.path.join(output_dir, 'catalog.xml')
catalog_root.write(cat_file, encoding='utf-8', xml_declaration=True, pretty_print=True)

# write out the md5sum cache file
print(md5sum_file)
md5sum_tree.write(md5sum_file, encoding='utf-8', xml_declaration=True, pretty_print=True)

# look for orphaned zip files
files = os.listdir(output_dir)
for file in files:
    if file.endswith('.zip') and not file in valid_zips:
        print("orphaned zip: %s" % file)

# process uploads
for up in config_node.findall('upload'):
    upload(up, output_dir)

# This entry point is declared in pyproject.toml and is necessary to have the
# script installed by the 'flightgear' Python package.
def dummyEntryPoint():
    sys.exit(0)
