#!/usr/bin/python3

# koji-gc: a garbage collection tool for Koji
# Copyright (c) 2007-2014 Red Hat, Inc.
#
# Authors:
#       Mike McLean <mikem@redhat.com>

import datetime
import fcntl
import fnmatch
import optparse
import os
import pprint
import smtplib
import sys
import time
import xmlrpc.client  # for Fault
from email.mime import text as MIMEText
from string import Template

import requests

import koji
import koji.policy
from koji.util import LazyDict, LazyValue, to_list, deprecated


def _(args):
    """Stub function for translation"""
    return args


def get_options():
    """process options from command line and config file"""

    usage = _("%prog [options]")
    parser = optparse.OptionParser(usage=usage)
    parser.add_option("-c", "--config-file", metavar="FILE",
                      default='/etc/koji-gc/koji-gc.conf',
                      help=_("use alternate configuration file"))
    parser.add_option("--keytab", help=_("specify a Kerberos keytab to use"))
    parser.add_option("--principal", help=_("specify a Kerberos principal to use"))
    parser.add_option("--runas", metavar="USER",
                      help=_("run as the specified user (requires special privileges)"))
    parser.add_option("--user", help=_("specify user"))
    parser.add_option("--password", help=_("specify password"))
    parser.add_option("--noauth", action="store_true", default=False,
                      help=_("do not authenticate"))
    parser.add_option("--network-hack", action="store_true", default=False,
                      help=optparse.SUPPRESS_HELP)  # no longer used
    parser.add_option("--cert", help=_("Client SSL certificate file for authentication"))
    parser.add_option("--ca", help=optparse.SUPPRESS_HELP)  # FIXME: remove in 1.24
    parser.add_option("--serverca", help=_("CA cert file that issued the hub certificate"))
    parser.add_option("-n", "--test", action="store_true", default=False,
                      help=_("test mode"))
    parser.add_option("-d", "--debug", action="store_true", default=False,
                      help=_("show debug output"))
    parser.add_option("--debug-xmlrpc", action="store_true", default=False,
                      help=_("show xmlrpc debug output"))
    parser.add_option("--smtp-host", metavar="HOST", default='localhost',
                      help=_("specify smtp server for notifications"))
    parser.add_option("--smtp-user", dest="smtp_user", metavar="USER",
                      help=_("specify smtp username for notifications"))
    parser.add_option("--smtp-pass", dest="smtp_pass", metavar="PASSWORD",
                      help=optparse.SUPPRESS_HELP)  # do not allow passwords on a command line
    parser.add_option("--no-mail", action='store_false', default=True, dest="mail",
                      help=_("don't send notifications"))
    parser.add_option("--send-mail", action='store_true', dest="mail",
                      help=_("send notifications"))
    parser.add_option("--email-domain", default="fedoraproject.org",
                      help=_("Email domain appended to Koji username for notifications"))
    parser.add_option("--from-addr", default="Koji Build System <buildsys@example.com>",
                      help=_("From address for notifications"))
    parser.add_option("--cc-addr", help=_("CC address for notifications (multiple)"),
                      action="append", metavar="EMAIL_ADDRESS")
    parser.add_option("--bcc-addr", help=_("BCC address for notifications (multiple)"),
                      action="append", metavar="EMAIL_ADDRESS")
    parser.add_option("--email-template", default="/etc/koji-gc/email.tpl",
                      help=_("notification template"))
    parser.add_option("--action", help=_("action(s) to take"))
    parser.add_option("--delay", metavar="INTERVAL", default='5 days',
                      help="time before eligible builds are placed in trashcan")
    parser.add_option("--grace-period", default='4 weeks', metavar="INTERVAL",
                      help="time that builds are held in trashcan")
    parser.add_option("--skip-main", action="store_true", default=False,
                      help=_("don't actually run main"))
    parser.add_option("--unprotected-keys", metavar="KEYS",
                      help=_("allow builds signed with these keys to be deleted"))
    parser.add_option("--tag-filter", "--tag", metavar="PATTERN", action="append",
                      help=_("Process only tags matching PATTERN when pruning"))
    parser.add_option("--ignore-tags", metavar="PATTERN", action="append",
                      help=_("Ignore tags matching PATTERN when pruning"))
    parser.add_option("--pkg-filter", "--pkg", "--package",
                      metavar="PATTERN", action='append',
                      help=_("Process only packages matching PATTERN"))
    parser.add_option("--bypass-locks", metavar="PATTERN", action="append",
                      help=_("Bypass locks for tags matching PATTERN"))
    parser.add_option("--purge", action="store_true", default=False,
                      help=_("When pruning, attempt to delete the builds that are untagged"))
    parser.add_option("--trashcan-tag", default='trashcan', metavar="TAG",
                      help=_("specify an alternate trashcan tag"))
    parser.add_option("--weburl", default="http://localhost/koji", metavar="URL",
                      help=_("url of koji web server (for use in notifications)"))
    parser.add_option("-s", "--server", help=_("url of koji XMLRPC server"))
    parser.add_option("--lock-file", help=_("koji-gc will wait while specified file exists. "
                                            "Default path is /run/user/<uid>/koji-gc.lock. "
                                            "For service usage /var/lock/koji-gc.lock is "
                                            "recommended."))
    parser.add_option("--exit-on-lock", action="store_true",
                      help=_("quit if --lock-file exists, don't wait"))
    # parse once to get the config file
    (options, args) = parser.parse_args()

    defaults = parser.get_default_values()

    cf = getattr(options, 'config_file')
    config = koji.read_config_files(cf)

    # List of values read from config file to update default parser values
    cfgmap = [
        # name, alias, type
        ['keytab', None, 'string'],
        ['principal', None, 'string'],
        ['runas', None, 'string'],
        ['user', None, 'string'],
        ['password', None, 'string'],
        ['noauth', None, 'boolean'],
        ['cert', None, 'string'],
        ['ca', None, 'string'],  # FIXME: remove in next major release
        ['serverca', None, 'string'],
        ['server', None, 'string'],
        ['weburl', None, 'string'],
        ['smtp_host', None, 'string'],
        ['smtp_user', None, 'string'],
        ['smtp_pass', None, 'string'],
        ['from_addr', None, 'string'],
        ['cc_addr', None, 'string'],
        ['bcc_addr', None, 'string'],
        ['email_template', None, 'string'],
        ['email_domain', None, 'string'],
        ['mail', None, 'boolean'],
        ['delay', None, 'string'],
        ['unprotected_keys', None, 'string'],
        ['grace_period', None, 'string'],
        ['trashcan_tag', None, 'string'],
        ['no_ssl_verify', None, 'boolean'],
        ['timeout', None, 'integer'],
        ['lock_file', None, 'string'],
        ['exit_on_lock', None, 'boolean'],
    ]
    for name, alias, type in cfgmap:
        if alias is None:
            alias = ('main', name)
        if config.has_option(*alias):
            if options.debug:
                print("Using option %s from config file" % (alias,))
            if type == 'integer':
                setattr(defaults, name, config.getint(*alias))
            elif type == 'boolean':
                setattr(defaults, name, config.getboolean(*alias))
            else:
                setattr(defaults, name, config.get(*alias))
    if config.has_option('main', 'cc_addr'):
        setattr(defaults, 'cc_addr', config.get('main', 'cc_addr').split(','))
    if config.has_option('main', 'bcc_addr'):
        setattr(defaults, 'bcc_addr', config.get('main', 'bcc_addr').split(','))
    # parse again with defaults
    (options, args) = parser.parse_args(values=defaults)
    options.config = config

    if options.ca:
        deprecated("--ca option is deprecated and will be removed in 1.24")

    # figure out actions
    actions = ('prune', 'trash', 'delete', 'salvage')
    if options.action:
        options.action = options.action.lower().replace(',', ' ').split()
        for x in options.action:
            if x not in actions:
                parser.error(_("Invalid action: %s") % x)
    else:
        options.action = ('delete', 'prune', 'trash')

    # split patterns for unprotected keys
    if options.unprotected_keys:
        options.unprotected_key_patterns = options.unprotected_keys.replace(',', ' ').split()
    else:
        options.unprotected_key_patterns = []

    # parse key aliases
    options.key_aliases = {}
    try:
        if config.has_option('main', 'key_aliases'):
            for line in config.get('main', 'key_aliases').splitlines():
                parts = line.split()
                if len(parts) < 2:
                    continue
                options.key_aliases[parts[0].upper()] = parts[1]
    except ValueError as e:
        print(e)
        parser.error(_("Invalid key alias data in config: %s") % config.get('main', 'key_aliases'))

    # parse time intervals
    for key in ('delay', 'grace_period'):
        try:
            value = getattr(options, key)
            value = parse_duration(value)
            setattr(options, key, value)
            if options.debug:
                print("%s: %s seconds" % (key, value))
        except ValueError:
            parser.error(_("Invalid time interval: %s") % value)

    # special handling for cert defaults
    cert_defaults = {
        'cert': '/etc/koji-gc/client.crt',
        'serverca': '/etc/koji-gc/serverca.crt',
    }
    for name in cert_defaults:
        if getattr(options, name, None) is None:
            fn = cert_defaults[name]
            if os.path.exists(fn):
                setattr(options, name, fn)

    template = getattr(options, 'email_template', None)
    if not template or not os.access(template, os.F_OK):
        parser.error(_("No such file: %s") % template)

    return options, args


def check_tag(name):
    """Check tag name against options and determine if we should process it

    The ignore option takes priority here.
    Returns True if we should process the tag, False otherwise
    """
    if options.ignore_tags:
        for pattern in options.ignore_tags:
            if fnmatch.fnmatch(name, pattern):
                return False
    if options.tag_filter:
        for pattern in options.tag_filter:
            if fnmatch.fnmatch(name, pattern):
                return True
        # doesn't match any pattern in filter
        return False
    else:
        # not ignored and no filter specified
        return True


def check_package(name):
    """Check package name against options and determine if we should process it

    Returns True if we should process the package, False otherwise
    """
    if options.pkg_filter:
        for pattern in options.pkg_filter:
            if fnmatch.fnmatch(name, pattern):
                return True
        # doesn't match any pattern in filter
        return False
    else:
        # no filter specified
        return True


time_units = {
    'second': 1,
    'minute': 60,
    'hour': 3600,
    'day': 86400,
    'week': 604800,
}
time_unit_aliases = [
    # [unit, alias, alias, ...]
    ['week', 'weeks', 'wk', 'wks'],
    ['hour', 'hours', 'hr', 'hrs'],
    ['day', 'days'],
    ['minute', 'minutes', 'min', 'mins'],
    ['second', 'seconds', 'sec', 'secs', 's'],
]


def parse_duration(str):
    """Parse time duration from string, returns duration in seconds"""
    ret = 0
    n = None
    unit = None

    def parse_num(s):
        try:
            return int(s)
        except ValueError:
            pass
        try:
            return float(s)
        except ValueError:
            pass
        return None
    for x in str.split():
        if n is None:
            n = parse_num(x)
            if n is not None:
                continue
            # perhaps the unit is appended w/o a space
            for names in time_unit_aliases:
                for name in names:
                    if x.endswith(name):
                        n = parse_num(x[:-len(name)])
                        if n is None:
                            continue
                        unit = names[0]
                        # combined at end
                        break
                if unit:
                    break
            else:
                raise ValueError("Invalid time interval: %s" % str)
        if unit is None:
            x = x.lower()
            for names in time_unit_aliases:
                for name in names:
                    if x == name:
                        unit = names[0]
                        break
                if unit:
                    break
            else:
                raise ValueError("Invalid time interval: %s" % str)
        ret += n * time_units[unit]
        n = None
        unit = None
    return ret


def error(msg=None, code=1):
    if msg:
        sys.stderr.write(msg + "\n")
        sys.stderr.flush()
    sys.exit(code)


def warn(msg):
    sys.stderr.write(msg + "\n")
    sys.stderr.flush()


def ensure_connection(session):
    try:
        ret = session.getAPIVersion()
    except requests.exceptions.ConnectionError:
        error(_("Error: Unable to connect to server"))
    if ret != koji.API_VERSION:
        warn(_("WARNING: The server is at API version %d and the client is at %d" %
               (ret, koji.API_VERSION)))


def activate_session(session):
    """Test and login the session is applicable"""
    global options
    if options.noauth:
        # skip authentication
        pass
    elif options.cert is not None and os.path.isfile(options.cert):
        # authenticate using SSL client cert
        session.ssl_login(options.cert, None, options.serverca, proxyuser=options.runas)
    elif options.user:
        # authenticate using user/password
        session.login()
    elif koji.reqgssapi:
        session.gssapi_login(principal=options.principal, keytab=options.keytab,
                             proxyuser=options.runas)
    if not options.noauth and not session.logged_in:
        error(_("Error: unable to log in, no authentication methods available"))
    ensure_connection(session)
    if options.debug:
        print("successfully connected to hub")


def send_warning_notice(owner_name, builds):
    if not options.mail:
        return
    if not builds:
        print("Warning: empty build list. No notice sent")
        return

    with open(options.email_template, 'r') as f:
        tpl = Template(f.read())

    fmt = """\
Build: %%(name)s-%%(version)s-%%(release)s
%s/buildinfo?buildID=%%(id)i""" % options.weburl
    middle = '\n\n'.join([fmt % b for b in builds])

    msg = MIMEText.MIMEText(tpl.safe_substitute(
        owner=owner_name,
        builds=middle,
    ))

    if len(builds) == 1:
        msg['Subject'] = "1 build marked for deletion"
    else:
        msg['Subject'] = "%i builds marked for deletion" % len(builds)
    msg['From'] = options.from_addr
    msg['To'] = "%s@%s" % (owner_name, options.email_domain)  # XXX!
    emails = [msg['To']]
    if options.cc_addr:
        msg['Cc'] = ','.join(options.cc_addr)
        emails += options.cc_addr
    if options.bcc_addr:
        emails += options.bcc_addr
    msg['X-Koji-Builder'] = owner_name
    if options.test:
        if options.debug:
            print(str(msg))
        else:
            print("Would have sent warning notice to %s" % emails)
    else:
        if options.debug:
            print("Sending warning notice to %s" % emails)
        try:
            s = smtplib.SMTP(options.smtp_host)
            if options.smtp_user is not None and options.smtp_pass is not None:
                s.login(options.smtp_user, options.smtp_pass)
            s.sendmail(msg['From'], emails, msg.as_string())
            s.quit()
        except Exception:
            print("FAILED: Sending warning notice to %s" % emails)


def main(args):
    activate_session(session)
    if not session.getTag(options.trashcan_tag, strict=False):
        error(_("Trashcan tag %s doesn't exist") % options.trashcan_tag)
    for x in options.action:
        globals()['handle_' + x]()


def handle_trash():
    print("Getting untagged builds...")
    untagged = session.untaggedBuilds()
    print("...got %i builds" % len(untagged))
    min_age = options.delay
    trashcan_tag = options.trashcan_tag
    # Step 1: place unreferenced builds into trashcan
    i = 0
    N = len(untagged)
    to_trash = []

    print("1st pass: blacklist")
    continuing = []
    for binfo in untagged:
        i += 1
        nvr = "%(name)s-%(version)s-%(release)s" % binfo
        binfo['nvr'] = nvr
        if not check_package(binfo['name']):
            if options.debug:
                print("[%i/%i] Skipping package: %s" % (i, N, nvr))
            continue
        continuing.append(binfo)

    print("2nd pass: references")
    i = 0
    mcall = koji.MultiCallSession(session, batch=1000)
    for binfo in continuing:
        mcall.buildReferences(binfo['id'], limit=10, lazy=True)
    for binfo, [refs] in zip(continuing, mcall.call_all()):
        i += 1
        nvr = binfo['nvr']
        # XXX - this is more data than we need
        #      also, this call takes waaaay longer than it should
        if refs.get('tags'):
            # must have been tagged just now
            print("[%i/%i] Build is tagged [?]: %s" % (i, N, nvr))
            continue
        if refs.get('rpms'):
            if options.debug:
                print("[%i/%i] Build has %i rpm references: %s" % (i, N, len(refs['rpms']), nvr))
                # pprint.pprint(refs['rpms'])
            continue
        if refs.get('archives'):
            if options.debug:
                print("[%i/%i] Build has %i archive references: %s" %
                      (i, N, len(refs['archives']), nvr))
                # pprint.pprint(refs['archives'])
            continue
        if refs.get('component_of'):
            if options.debug:
                print("[%i/%i] Build is a component of archives: %s" % (i, N, nvr))
            continue
        ts = refs['last_used']
        if ts:
            # work around server bug
            if isinstance(ts, list):
                ts = ts[0]
            # XXX - should really check time server side
            if options.debug:
                print("[%i/%i] Build has been used in a buildroot: %s" % (i, N, nvr))
                print("Last_used: %s" % datetime.datetime.fromtimestamp(ts).isoformat())
            age = time.time() - ts
            if age < min_age:
                continue
        # see how long build has been untagged
        history = session.queryHistory(build=binfo['id'])['tag_listing']
        age = None
        binfo2 = None
        if not history:
            # never tagged, we'll have to use the build create time
            binfo2 = session.getBuild(binfo['id'])
            ts = binfo2.get('creation_ts')
            if ts is None:
                # older api with no good way to get a proper timestamp for
                # a build, so we have the following hack
                task_id = binfo2.get('task_id')
                if task_id:
                    tinfo = session.getTaskInfo(task_id)
                    if tinfo['completion_ts']:
                        age = time.time() - tinfo['completion_ts']
            else:
                age = time.time() - ts
        else:
            tagged = False
            last = history[0]
            for h in history:
                if not h['revoke_event']:
                    tagged = True
                    break
                if h['revoke_event'] > last['revoke_event']:
                    last = h
            if tagged:
                # this might happen if the build was tagged just now
                print("[%i/%i] Warning: build not untagged: %s" % (i, N, nvr))
                continue
            age = time.time() - last['revoke_ts']
        if age is not None and age < min_age:
            if options.debug:
                print("[%i/%i] Build untagged only recently: %s" % (i, N, nvr))
            continue
        # check build signatures
        keys = get_build_sigs(binfo['id'], cache=True)
        if keys and options.debug:
            print("Build: %s, Keys: %s" % (nvr, keys))
        if protected_sig(keys):
            print("Skipping build %s. Keys: %s" % (nvr, keys))
            continue

        # ok, go ahead add it to the list
        if binfo2 is None:
            binfo2 = session.getBuild(binfo['id'])
        print("[%i/%i] Adding build to trash list: %s" % (i, N, nvr))
        to_trash.append(binfo2)

    # process to_trash
    # group by owner so we can reduce the number of notices
    by_owner = {}
    for binfo in to_trash:
        by_owner.setdefault(binfo['owner_name'], []).append(binfo)
    owners = sorted(to_list(by_owner.keys()))
    mcall = koji.MultiCallSession(session, batch=1000)
    for owner_name in owners:
        builds = sorted([(b['nvr'], b) for b in by_owner[owner_name]])
        send_warning_notice(owner_name, [x[1] for x in builds])
        for nvr, binfo in builds:
            if options.test:
                print("Would have moved to trashcan: %s" % nvr)
            else:
                if options.debug:
                    print("Moving to trashcan: %s" % nvr)
                # figure out package owner
                count = {}
                for pkg in session.listPackages(pkgID=binfo['name']):
                    count.setdefault(pkg['owner_id'], 0)
                    count[pkg['owner_id']] += 1
                if not count:
                    print("Warning: no owner for %s, using build owner" % nvr)
                    # best we can do currently
                    owner = binfo['owner_id']
                else:
                    owner = max([(n, k) for k, n in count.items()])[1]
                mcall.packageListAdd(trashcan_tag, binfo['name'], owner)
                mcall.tagBuildBypass(trashcan_tag, binfo['id'], force=True)
    # run all packageListAdd/tagBuildBypass finally
    mcall.call_all()


def protected_sig(keys):
    """Check list of keys and see if any are protected

    returns True if ANY are protected (not on unprotected list)
    returns False if ALL are unprotected
    """
    for key in keys:
        if not key:
            continue
        if not sigmatch(key, options.unprotected_key_patterns):
            # this key is protected
            return True
    return False


def handle_salvage():
    """Reclaim builds from trashcan

    Check builds in trashcan for new tags or references and salvage them
    (remove trashcan tag) if appropriate.

    The delete action also does this, but this is for when you want to
    run this action only."""
    return handle_delete(just_salvage=True)


def salvage_build(binfo):
    """Removes trashcan tag from a build and prints a message"""
    if options.test:
        print("Would have untagged from trashcan: %(nvr)s" % binfo)
    else:
        if options.debug:
            print("Untagging from trashcan: %(nvr)s" % binfo)
        session.untagBuildBypass(options.trashcan_tag, binfo['id'], force=True)


def handle_delete(just_salvage=False):
    """Delete builds that have been in the trashcan for long enough

    If just_salvage is True, goes into salvage mode. In salvage mode it only
    reclaims referenced builds from the trashcan, it does not perform any
    deletes
    """
    print("Getting list of builds in trash...")
    trashcan_tag = options.trashcan_tag
    trash = sorted([(b['nvr'], b) for b in session.listTagged(trashcan_tag)])
    print("...got %i builds" % len(trash))
    # XXX - it would be better if there were more appropriate server calls for this
    grace_period = options.grace_period
    import time

    print("1st pass: blacklist")
    continuing = []
    for nvr, binfo in trash:
        if not check_package(binfo['name']):
            if options.debug:
                print("Skipping package: %s" % nvr)
            continue
        continuing.append((nvr, binfo))

    print("2nd pass: tags")
    continuing, trash = [], continuing
    mcall = koji.MultiCallSession(session, batch=1000)
    for nvr, binfo in trash:
        mcall.listTags(build=binfo['id'], perms=False)
    for (nvr, binfo), [tags] in zip(trash, mcall.call_all()):
        # see if build has been tagged elsewhere
        tags = [t['name'] for t in tags if t['name'] != trashcan_tag]
        if tags:
            print("Build %s tagged elsewhere: %s" % (nvr, tags))
            salvage_build(binfo)
            continue
        continuing.append((nvr, binfo))

    print("3rd pass: signatures")
    continuing, trash = [], continuing
    for nvr, binfo in trash:
        # check build signatures
        keys = get_build_sigs(binfo['id'], cache=False)
        if keys and options.debug:
            print("Build: %s, Keys: %s" % (nvr, keys))
        if protected_sig(keys):
            print("Salvaging signed build %s. Keys: %s" % (nvr, keys))
            salvage_build(binfo)
            continue
        if just_salvage:
            # skip the rest when salvaging
            continue
        continuing.append((nvr, binfo))

    print("4th pass: history")
    continuing, trash = [], continuing
    for nvr, binfo in trash:
        # determine how long this build has been in the trashcan
        mcall.queryHistory(build=binfo['id'], tag=trashcan_tag)

    for (nvr, binfo), [history] in zip(trash, mcall.call_all()):
        current = [x for x in history['tag_listing'] if x['active']]
        if not current:
            # untagged just now?
            print("Warning: history missing for %s" % nvr)
            pprint.pprint(binfo)
            pprint.pprint(history)
            continue
        assert len(current) == 1  # see db constraint
        current = current[0]
        age = time.time() - current['create_ts']
        if age < grace_period:
            if options.debug:
                print("Skipping build %s, age=%i" % (nvr, age))
            continue
        continuing.append(binfo)

    print("5th pass: deletion")
    for binfo in continuing:
        # go ahead and delete
        if options.test:
            print("Would have deleted build from trashcan: %s" % binfo['nvr'])
        else:
            print("Deleting build: %s" % binfo['nvr'])
            mcall.untagBuildBypass(trashcan_tag, binfo['id'])
            mcall.deleteBuild(binfo['id'])

    for binfo, result in zip(continuing, mcall.call_all()):
        if isinstance(result, dict):
            print("Warning: deletion failed: %s" % result['faultString'])
            # TODO - log details for delete failures


class TagPruneTest(koji.policy.MatchTest):
    name = 'tag'
    field = 'tagname'


class PackagePruneTest(koji.policy.MatchTest):
    name = 'package'
    field = 'pkgname'


class VolumePruneTest(koji.policy.MatchTest):
    name = 'volume'
    field = 'volname'


class SigPruneTest(koji.policy.BaseSimpleTest):
    name = 'sig'

    def run(self, data):
        # true if any of the keys match any of the patterns
        patterns = self.str.split()[1:]
        for key in data['keys']:
            if sigmatch(key, patterns):
                return True
        return False


def sigmatch(key, patterns):
    """Test whether a key id matches any of the given patterns

    Supports key aliases
    """
    if not isinstance(patterns, (tuple, list)):
        patterns = (patterns,)
    for pat in patterns:
        if fnmatch.fnmatch(key, pat):
            return True
        alias = options.key_aliases.get(key.upper())
        if alias is not None and fnmatch.fnmatch(alias, pat):
            return True
    return False


class OrderPruneTest(koji.policy.CompareTest):
    name = 'order'
    field = 'order'
    allow_float = False


class AgePruneTest(koji.policy.BaseSimpleTest):
    name = 'age'
    cmp_idx = koji.policy.CompareTest.operators

    def __init__(self, str):
        """Read the test parameters from string"""
        super(AgePruneTest, self).__init__(str)
        self.cmp, value = str.split(None, 2)[1:]
        self.func = self.cmp_idx.get(self.cmp, None)
        if self.func is None:
            raise Exception("Invalid comparison in test: %s" % str)
        self.span = parse_duration(value)

    def run(self, data):
        return self.func(time.time() - data['ts'], self.span)


def read_policies(fn=None):
    """Read tag gc policies from file

    The expected format as follows
        test [params] [&& test [params] ...] :: (keep|untag|skip)
    """
    fo = open(fn, 'r')
    tests = koji.policy.findSimpleTests(globals())
    ret = koji.policy.SimpleRuleSet(fo, tests)
    fo.close()
    return ret


def scan_policies(str):
    """Read tag gc policies from a string

    The expected format as follows
        test [params] [&& test [params] ...] :: (keep|untag|skip)
    """
    tests = koji.policy.findSimpleTests(globals())
    return koji.policy.SimpleRuleSet(str.splitlines(), tests)


build_sig_cache = {}


def get_build_sigs(build, cache=False):
    if cache and build in build_sig_cache:
        return build_sig_cache[build]
    rpms = session.listRPMs(buildID=build)
    keys = {}
    if not rpms:
        # for non-rpm builds we have no easy way of checking signatures
        ret = build_sig_cache[build] = []
        return ret
    else:
        # TODO - multicall helps, but it might be good to have a more robust server-side call
        session.multicall = True
        for rpminfo in rpms:
            session.queryRPMSigs(rpm_id=rpminfo['id'])
        for rpminfo, [sigs] in zip(rpms, session.multiCall()):
            for sig in sigs:
                if sig['sigkey']:
                    keys.setdefault(sig['sigkey'], 1)
    ret = build_sig_cache[build] = to_list(keys.keys())
    return ret


def handle_prune():
    """Untag old builds according to policy

    If purge is True, will also attempt to delete the pruned builds afterwards
    """
    # read policy
    if not options.config or not options.config.has_option('prune', 'policy'):
        print("Skipping prune step. No policies available.")
        return
    # policies = read_policies(options.policy_file)
    policies = scan_policies(options.config.get('prune', 'policy'))
    for action in policies.all_actions():
        if action not in ("keep", "untag", "skip"):
            raise Exception("Invalid action: %s" % action)
    if options.debug:
        pprint.pprint(policies.ruleset)
    # get tags
    tags = session.listTags(perms=False, queryOpts={'order': 'name'})
    untagged = {}
    build_ids = {}
    for taginfo in tags:
        tagname = taginfo['name']
        if tagname == options.trashcan_tag:
            if options.debug:
                print("Skipping trashcan tag: %s" % tagname)
            continue
        if not check_tag(tagname):
            # if options.debug:
            #    print("skipping tag due to filter: %s" % tagname)
            continue
        bypass = False
        if taginfo['locked']:
            if options.bypass_locks:
                for pattern in options.bypass_locks:
                    if fnmatch.fnmatch(tagname, pattern):
                        bypass = True
                        break
            if bypass:
                print("Bypassing lock on tag: %s" % tagname)
            else:
                if options.debug:
                    print("skipping locked tag: %s" % tagname)
                continue
        if options.debug:
            print("Pruning tag: %s" % tagname)
        # get builds
        history = session.queryHistory(tag=taginfo['id'], active=True)['tag_listing']
        history = sorted(history, key=lambda x: -x['create_ts'])
        if not history:
            if options.debug:
                print("No history for %s" % tagname)
            continue
        pkghist = {}
        for h in history:
            if taginfo['maven_include_all'] and h.get('maven_build_id'):
                pkghist.setdefault(h['name'] + '-' + h['version'], []).append(h)
            else:
                pkghist.setdefault(h['name'], []).append(h)
        pkgs = sorted(to_list(pkghist.keys()))
        for pkg in pkgs:
            if not check_package(pkg):
                # if options.debug:
                #    print("skipping package due to filter: %s" % pkg)
                continue
            if options.debug:
                print(pkg)
            hist = pkghist[pkg]
            # these are the *active* history entries for tag/pkg
            skipped = 0
            for order, entry in enumerate(hist):
                # get sig data
                nvr = "%(name)s-%(version)s-%(release)s" % entry
                data = {
                    'tagname': tagname,
                    'pkgname': pkg,
                    'order': order - skipped,
                    'ts': entry['create_ts'],
                    'nvr': nvr,
                }
                data = LazyDict(data)
                data['keys'] = LazyValue(get_build_sigs, (entry['build_id'],), {'cache': True})
                data['volname'] = LazyValue(lambda x: session.getBuild(x).get('volume_name'),
                                            (entry['build_id'],), cache=True)
                build_ids[nvr] = entry['build_id']
                action = policies.apply(data)
                if action is None:
                    if options.debug:
                        print("No policy for %s (%s)" % (nvr, tagname))
                if action == 'skip':
                    skipped += 1
                if options.debug:
                    print(policies.last_rule())
                    print("%s: %s (%s, %i)" % (action, nvr, tagname, order))
                if action == 'untag':
                    if options.test:
                        print("Would have untagged %s from %s" % (nvr, tagname))
                        untagged.setdefault(nvr, {})[tagname] = 1
                    else:
                        print("Untagging build %s from %s" % (nvr, tagname))
                        try:
                            session.untagBuildBypass(taginfo['id'], entry['build_id'],
                                                     force=bypass)
                            untagged.setdefault(nvr, {})[tagname] = 1
                        except (xmlrpc.client.Fault, koji.GenericError) as e:
                            print("Warning: untag operation failed: %s" % e)
                            pass
                # if action == 'keep' do nothing
    if options.purge and untagged:
        print("Attempting to purge %i builds" % len(untagged))
        for nvr in untagged:
            build_id = build_ids[nvr]
            tags = [t['name'] for t in session.listTags(build_id, perms=False)]
            if options.test:
                # filted out the tags we would have dropped above
                tags = [t for t in tags if t not in untagged[nvr]]
            if tags:
                # still tagged somewhere
                print("Skipping %s, still tagged: %s" % (nvr, tags))
                continue
            # check cached sigs first to save a little time
            if build_id in build_sig_cache:
                keys = build_sig_cache[build_id]
                if protected_sig(keys):
                    print("Skipping %s, signatures: %s" % (nvr, keys))
                    continue
            # recheck signatures in case build was signed during run
            keys = get_build_sigs(build_id, cache=False)
            if protected_sig(keys):
                print("Skipping %s, signatures: %s" % (nvr, keys))
                continue

            if options.test:
                print("Would have deleted build: %s" % nvr)
            else:
                print("Deleting untagged build: %s" % nvr)
                try:
                    session.deleteBuild(build_id, strict=True)
                except (xmlrpc.client.Fault, koji.GenericError) as e:
                    print("Warning: deletion failed: %s" % e)
                    # server issue
                    pass


if __name__ == "__main__":

    options, args = get_options()

    session_opts = koji.grab_session_options(options)
    session = koji.ClientSession(options.server, session_opts)

    rv = 0
    try:
        lock_fd = None
        if not options.lock_file:
            options.lock_file = '/run/user/%d/koji-gc.lock' % os.getuid()
        # acquire lock file
        while not lock_fd:
            # fail, if it is completely inaccessible
            lock_fd = os.open(options.lock_file, os.O_CREAT | os.O_RDWR)
            try:
                fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
                break
            except (IOError, OSError):
                if options.exit_on_lock:
                    try:
                        session.logout()
                    except Exception:
                        pass
                    sys.exit(1)
            os.close(lock_fd)
            lock_fd = None
            if options.debug:
                print("Waiting on lock: %s" % options.lock_file)
            time.sleep(10)

        if not options.skip_main:
            rv = main(args)
        if not rv:
            rv = 0

        if lock_fd:
            # release lock file
            fcntl.flock(lock_fd, fcntl.LOCK_UN)
            os.close(lock_fd)
    except KeyboardInterrupt:
        pass
    except SystemExit:
        rv = 1
    # except:
    #    if options.debug:
    #        raise
    #    else:
    #        exctype, value = sys.exc_info()[:2]
    #        rv = 1
    #        print("%s: %s" % (exctype, value))
    try:
        session.logout()
    except Exception:
        pass
    if not options.skip_main:
        sys.exit(rv)
