#!/usr/bin/env python

"""
Review helper script for openQA.

# Inspiration

We want to gather information about the build status and condense this into a
review report.

E.g.

1. Go to the openQA Dashboard and select the latest build of the product you
want to review.

2. Walk through all red testcases for the product for all arches.

 - fix the needle
 - report a bug against openQA
 - report a bug against the product


Also review still failing test cases.

3. Add a comment to the overview page of the reviewed product using the
template generated by this script


# What it does

On calling the script it parses openQA status reports from openQA server
webpages and generates markdown text usable as template for review reports.

So far it is save to call it as it does not use or need any kind of
authentication and only reads the webpage. No harm should be done :-)

## feature list

 - Command line options with different modes, e.g. for markdown report generation
 - Strip optional "Build" from build number when searching for last reviewed
 - Support differing tests in test rows
 - Loading and saving of cache files (e.g. for testing)
 - Skip over '(reference ...)' searching for last reviewed
 - Yield last finished in case of no reviewed found in comments
 - Add proper handling for non-number build number, e.g. for 'SLE HA'
 - Tests to ensure 100% statement and branch coverage
 - Coverage analysis and test
 - Option to compare build against last reviewed one from comments section
 - Extended '--job-groups' to also accept regex terms
 - Add notice in report if architectures are not run at all
 - Support for explicit selection of builds not in job group display anymore
 - Add optional link to previous build for comparison for new issues
 - Option to specify builds for comparison
 - Support both python version 2 and 3
 - Human friendly progress notification and wait spinner
 - Accept multiple entries for '--job-group(-urls)'
 - Ensure report entries are in same alphabetical order with OrderedDict
 - tox.ini: Local tests, doctests, check with flake8
 - Generate version based on git describe
 - Add support to parse all job groups


# How to use

Just call it and see what happens or call this file with option '--help'.


# Design decisions

The script was designed to be a webscraping script for the following reasons

 * It should as closely resemble what the human review user encounters and not
   use any "hidden API" magic
 * "proof of concept": Show that it is actually possible using webscraping on
   a clean web design as openQA provides :-)
 * Do not rely on the annoyingly slow login and authentication redirect
   mechanism used on openqa.opensuse.org as well as openqa.opensuse.org


Alternatives could have been and still are for further extensions or reworks:

 * Use of https://github.com/os-autoinst/openQA-python-client and extend on
   that
 * Directly include all necessary meta-reports as part of openQA itself
 * Use REST or websockets API instead of webscraping


"""

# Python 2 and 3: easiest option
# see http://python-future.org/compatible_idioms.html
from __future__ import absolute_import
from future.standard_library import install_aliases  # isort:skip to keep 'install_aliases()'
install_aliases()
from future.utils import iteritems

import argparse
import datetime
import logging
import os.path
import re
import sys
import json
from collections import defaultdict, OrderedDict
from configparser import ConfigParser, NoSectionError, NoOptionError  # isort:skip can not make isort happy here
from requests.exceptions import HTTPError
from string import Template
from urllib.parse import quote, unquote, urljoin, urlencode, splitquery, parse_qs

from bs4 import BeautifulSoup
from sortedcontainers import SortedDict

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

from openqa_review.browser import Browser, DownloadError, add_load_save_args  # isort:skip


# treat humanfriendly as optional dependency
humanfriendly_available = False
try:
    from humanfriendly import AutomaticSpinner
    from humanfriendly.text import pluralize
    humanfriendly_available = True
except ImportError:  # pragma: no cover
    def pluralize(_1, _2, plural):
        return plural


# minimum number of days an issue is unchanged before putting a reminder comment
MIN_DAYS_UNCHANGED = 14

logging.basicConfig()
log = logging.getLogger(sys.argv[0] if __name__ == "__main__" else __name__)
logging.captureWarnings(True)  # see https://urllib3.readthedocs.org/en/latest/security.html#disabling-warnings

config = None


CONFIG_PATH = os.path.expanduser('~') + '/.openqa_reviewrc'
CONFIG_USAGE = """
You are missing the mandatory configuration file with the proper format.
Example:
[product_issues]
system = bugzilla
# if username and password are not defined here, they will be stored in your
# local keyring if found
#username = user
#password = secret
base_url = https://apibugzilla.suse.com
report_url = https://bugzilla.suse.com

# for correct generation of issue reporting links add mappings from openQA
# group IDs to product names in the corresponding issue tracker, e.g.
# necessary for bugzilla
[product_issues:https://openqa.opensuse.org:product_mapping]
25 = openSUSE Tumbleweed

[product_issues:http://openqa.opensuse.org:component_mapping]
installation-bootloader = Bootloader

[test_issues]
system = redmine
#api_key = 0123456789ABCDEF
report_url = https://progress.opensuse.org/projects/openqatests/issues/new
"""


openqa_review_report_product_template = Template("""
**Date:** $now
**Build:** $build
$common_issues
---
$arch_report
""")  # noqa: W291  # ignore trailing whitespace for forced line breaks

todo_review_template = Template("""
**TODO: review**
$new_issues$existing_issues""")

# TODO don't display sections if empty
openqa_review_report_arch_template = Template("""
**Arch:** $arch
**Status: $status_badge**
$new_product_issues$existing_product_issues$new_openqa_issues$existing_openqa_issues$todo_issues""")

openqa_issue_comment = Template("""
This is an autogenerated message for openQA integration by the openqa_review script:

This bug is still referenced in a failing openQA test: $name
$url
""")

status_badge_str = {
    'GREEN': '<span style="color: green;">Green</span>',
    'AMBER': '<span style="color: #FFBF00;">Amber</span>',
    'RED': '<span style="color: red;">Red</span>',
}


class NotEnoughBuildsError(Exception):
    """Not enough finished builds found."""
    pass


def parse_summary(details):
    """Parse and return build summary as dict."""
    return {i.previous.strip().rstrip(':').lower(): int(i.text) for i in details.find(id="summary").find_all(class_="badge")}


change_state = {
    ('result_passed', 'result_failed'): 'NEW_ISSUE',
    ('result_softfailed', 'result_failed'): 'NEW_ISSUE',
    ('result_passed', 'result_softfailed'): 'NEW_SOFT_ISSUE',
    ('result_failed', 'result_passed'): 'FIXED',  # fixed, maybe spurious, false positive
    ('result_softfailed', 'result_passed'): 'FIXED',
    ('result_failed', 'result_failed'): 'STILL_FAILING',  # still failing or partial improve, partial degrade
    ('result_softfailed', 'result_softfailed'): 'STILL_SOFT_FAILING',
    ('result_failed', 'result_softfailed'): 'IMPROVED',
    ('result_passed', 'result_passed'): 'STABLE',  # ignore or crosscheck if not fals positive
}

soft_fail_states = ['STILL_SOFT_FAILING', 'NEW_SOFT_ISSUE', 'IMPROVED']

interesting_states_names = [i for i in set(change_state.values()) if i != 'STABLE'] + ['INCOMPLETE']

issue_tracker = {  # pragma: no branch
    'bsc': lambda i: 'https://bugzilla.suse.com/show_bug.cgi?id=%s' % i,
    'boo': lambda i: 'https://bugzilla.opensuse.org/show_bug.cgi?id=%s' % i,
    'poo': lambda i: 'https://progress.opensuse.org/issues/%s' % i,
    'bgo': lambda i: 'https://bugzilla.gnome.org/show_bug.cgi?id=%s' % i,
}


def status(entry):
    """Return test status from entry, e.g. 'result_passed'."""
    return [s for s in entry.i['class'] if re.search('(state|result)_', s)][0]


def get_build_nr(url):
    return unquote(re.search('build=([^&]*)', url).groups()[0])


def get_failed_needles(m):
    return [str(i.text) for i in BeautifulSoup(m['title'], 'html.parser').find_all('li')] if m.get('title') else []


def get_test_details(entry):
    failedmodules = entry.find_all(class_='failedmodule')

    def find_module_href(m):
        # optional fallback to old openQA code, 4.5, pre bootstrap4
        try:
            return m['href']
        except KeyError:
            return m.a['href']

    return {'href': entry.a['href'],
            'failedmodules': [{'href': find_module_href(m), 'name': m.text.strip(), 'needles': get_failed_needles(m)} for m in failedmodules]
            }


def get_test_bugref(entry):
    bugref = entry.find(id=re.compile('^bug-'))
    if not bugref:
        return {}
    # work around openQA providing incorrect URLs (e.g. following whitespace)
    return {'bugref': re.search('\S+#([0-9]+)', bugref.i['title']).group(),
            'bugref_href': bugref.a['href'].strip()
            }


def get_state(cur, prev_dict):
    """Return change_state for 'previous' and 'current' test status html-td entries."""
    # TODO instead of just comparing the overall state we could check if
    # failing needles differ
    try:
        prev = prev_dict[cur['id']]
        state_dict = {'state': change_state[(status(prev), status(cur))]}
        # add more details, could be skipped if we don't have details
        state_dict.update({'prev': {'href': prev.find('a')['href']}})
    except KeyError:
        # if there is no previous or it was never completed we assume passed to mark new failing test as 'NEW_ISSUE'
        state_dict = {'state': change_state.get(('result_passed', status(cur)), 'INCOMPLETE')}
    state_dict.update(get_test_details(cur))
    state_dict.update(get_test_bugref(cur))
    return cur['id'], state_dict


def get_arch_state_results(arch, current_details, previous_details, output_state_results=False):
    result_re = re.compile(arch + '_')
    test_results = current_details.find_all('td', id=result_re)
    test_results_previous = previous_details.find_all('td', id=result_re)
    # find differences from previous to current (result_X)
    test_results_dict = {i['id']: i for i in test_results}
    test_results_previous_dict = {i['id']: i for i in test_results_previous if i['id'] in test_results_dict.keys()}
    states = SortedDict(get_state(v, test_results_previous_dict) for k, v in iteritems(test_results_dict))
    # intermediate step:
    # - print report of differences
    interesting_states = SortedDict({k.split(arch + '_')[1]: v for k, v in iteritems(states) if v['state'] != 'STABLE'})
    if output_state_results:
        print("arch: %s" % arch)
        for state in interesting_states_names:
            print("\n%s:\n\t%s\n" % (state, ', '.join(k for k, v in iteritems(interesting_states) if v['state'] == state)))
    return interesting_states


def absolute_url(root, v):
    return urljoin(root, str(v['href']))


def progress_browser_factory(args):
    return Browser(args, '', auth=(
        config.get('test_issues', 'api_key'),
        "foobar",
    ))


def bugzilla_browser_factory(args):
    return Browser(args, config.get('product_issues', 'base_url'), auth=(
        config.get('product_issues', 'username'),
        config.get('product_issues', 'password'),
    ))


def issue_listing(header, issues, show_empty=True):
    r"""
    Generate one issue listing section.

    @param header: Header string for section
    @param issues: List of IssueEntry objects
    @param show_empty: show empty sections if True and issues are an empty string

    >>> issue_listing('***new issues:***', 'None')
    '\n***new issues:***\n\nNone\n'
    >>> issue_listing('***Common issues:***', '')
    '\n***Common issues:***\n\n\n'
    >>> issue_listing('***no issues***', '', show_empty=False)
    ''
    """
    if not show_empty and len(issues) == 0:
        return ''
    return '\n' + header + '\n\n' + ''.join(map(str, issues)) + '\n'


def common_issues(issues, show_empty=True):
    if not show_empty and issues == '':
        return ''
    return '\n' + '**Common issues:**' + '\n' + issues + '\n'


def issue_type(bugref):
    return 'openqa' if re.match('poo#', bugref) else 'product'


def issue_state(result_list):
    # if any result was still failing the issue is regarded as existing
    return 'existing' if [i for i in result_list if re.match('(STILL|IMPROVED)', i['state'])] else 'new'


def get_results_by_bugref(results, args):
    include_tags = ['STILL_FAILING', 'NEW_ISSUE']
    if args.include_softfails:
        include_tags += soft_fail_states

    # plain for-loop with append is most efficient: https://stackoverflow.com/questions/11276473/append-to-a-dict-of-lists-with-a-dict-comprehension
    results_by_bugref = defaultdict(list)
    for k, v in iteritems(results):
        if not re.match('(' + '|'.join(include_tags) + ')', v['state']):
            continue
        key = v['bugref'] if (args.bugrefs and 'bugref' in v and v['bugref']) else 'todo'
        results_by_bugref[key].append(dict(v, **{'name': k}))
    return results_by_bugref


def set_status_badge(states):
    # TODO pretty arbitrary
    if states.count('NEW_ISSUE') == 0 and states.count('STILL_FAILING') <= 1:
        return 'GREEN'
    # still failing and soft issues allowed; TODO also arbitrary, just adjusted to test set
    elif states.count('NEW_ISSUE') == 0 and states.count('STILL_FAILING') <= 5:
        return 'AMBER'
    else:
        return 'RED'


def find_builds(builds, running_threshold=0):
    """Find finished builds, ignore still running or empty."""
    threshold = float(running_threshold) if running_threshold is not None else 0

    # filter out empty builds
    def non_empty(r):
        return r['total'] != 0 and r['total'] > r['skipped'] and not ('build' in r.keys() and r['build'] is None)
    builds = {build: result for build, result in iteritems(builds) if non_empty(result)}

    finished = {build: result for build, result in iteritems(builds) if not result['unfinished'] or
                (100 * float(result['unfinished']) / result['total']) <= threshold}

    log.debug("Found the following finished non-empty builds: %s" % ', '.join(finished.keys()))
    if len(finished) < 2:
        raise NotEnoughBuildsError("not enough finished builds found")
    assert len(finished.keys()) >= 2
    return finished.keys()


def find_last_reviewed_build(comments):
    """Find last reviewed build within job group comments."""
    # Could also find previous one with a comment on the build status,
    # i.e. a reviewed finished build
    # The build number itself might be prefixed with a redundant 'Build' which we ignore
    build_re = re.compile('[bB]uild:(\*\*)? *(Build)?([\w@.]*)(.*reference.*)?(\*\*)?\r\n')
    # Assuming the most recent with a build number also has the most recent review
    for c in reversed(comments):
        match = build_re.search(c['text'])
        if match:
            last_reviewed = match.group(3)
            break
    return last_reviewed


def get_build_urls_to_compare(browser, job_group_url, builds='', against_reviewed=None, running_threshold=0):
    """
    From the job group page get URLs for the builds to compare.

    @param browser: A browser instance
    @param job_group_url: forwarded to browser instance
    @param builds: Builds for which URLs should be retrieved as comma-separated pair, w/o the word 'Build'
    @param against_reviewed: Alternative to 'builds', which build to retrieve for comparison with last reviewed, can be 'last' to automatically select the last
           finished
    @param running_threshold: Threshold of which percentage of jobs may still be running for the build to be considered 'finished' anyway
    """
    job_group = browser.get_json('%s.json' % job_group_url)

    def get_group_result():
        try:
            results_list = job_group['build_results']
            return {i['build']: i for i in results_list}
        except KeyError:
            log.debug("Reverting to old openQA behaviour before openQA#9b50b22")
            return job_group['result']

    def build_url(build):
        r = get_group_result()
        b = r.get(build, next(iter(r.values())))
        build = b.get('build', build)
        # openQA introduced multi-distri support for the job groups with openQA#037ffd33
        distri_str = 'distri=%s' % b['distri'] if 'distri' in b.keys() else 'distri=' + '&distri='.join(sorted(b['distris'].keys()))
        return '/tests/overview?%s&version=%s&build=%s&groupid=%i' % (distri_str, b['version'], quote(build), job_group['group']['id'])

    finished_builds = find_builds(get_group_result(), running_threshold)
    # find last finished and previous one
    builds_to_compare = sorted(finished_builds, reverse=True)[0:2]

    if builds:
        # User has to be careful here. A page for non-existant builds is always
        # existant.
        builds_to_compare = builds.split(',')
        if min(map(len, builds_to_compare)) < 4:
            log.warning("A build number of at least four digits is expected with leading zero, expect weird results.")  # pragma: no cover
    elif against_reviewed:
        try:
            last_reviewed = find_last_reviewed_build(job_group['comments'])
            log.debug("Comparing specified build %s against last reviewed %s" % (against_reviewed, last_reviewed))
            build_to_review = max(finished_builds) if against_reviewed == 'last' else against_reviewed
            assert len(build_to_review) <= len(last_reviewed) + 1, "build_to_review and last_reviewed differ too much to make sense"
            builds_to_compare = build_to_review, last_reviewed
        except (NameError, AttributeError, IndexError):
            log.info("No last reviewed build found for URL %s, reverting to two last finished" % job_group_url)

    log.debug("Comparing build %s against %s" % (builds_to_compare[0], builds_to_compare[1]))
    current_url, previous_url = map(build_url, builds_to_compare)
    log.debug("Found two build URLS, current: %s previous: %s" % (current_url, previous_url))
    return current_url, previous_url


def get_failed_module_details_for_report(f):
    try:
        failed_module = f['failedmodules'][0]
    except IndexError:
        log.debug("%s does not have failed module, taking complete job." % f['href'])
        name = ''
        url = f['href']
        details = ''
    else:
        name = failed_module['name']
        url = failed_module['href']
        details = '\nwith failed needles: %s' % failed_module['needles']
    return name, url, details


def issue_report_link(root_url, f, test_browser=None):
    """Generate a bug reporting link for the current issue."""
    # always select the first failed module.
    # It might not be the fatal one but better be safe and assume the first
    # failed module introduces a problem in the whole job

    test_details_page = test_browser.get_soup(f['href'])
    current_build_overview = splitquery(test_details_page.find(id='current-build-overview').a['href'])
    overview_params = parse_qs(current_build_overview[-1])
    group = overview_params['groupid'][0]
    build = overview_params['build'][0]
    scenario_div = test_details_page.find(class_='previous').div.div
    scenario = re.findall('Results for (.*) \(', scenario_div.text)[0]
    latest_link = absolute_url(root_url, scenario_div.a)
    module, url, details = get_failed_module_details_for_report(f)
    previous_results = test_details_page.find(id='previous_results', class_='overview').find_all('tr')[1:]
    previous_results_list = [(i.td['id'], {'status': status(i),
                                           'details': get_test_details(i),
                                           'build': i.find(class_='build').text}) for i in previous_results]
    good = re.compile('(?<=_)(passed|softfailed)')

    def build_link(v):
        return '[%s](%s)' % (v['build'], absolute_url(root_url, v['details']))
    first_known_bad = build + ' (current job)'
    last_good = '(unknown)'
    for k, v in previous_results_list:
        if good.search(v['status']):
            last_good = build_link(v)
            break
        first_known_bad = build_link(v)
    description = """### Observation

openQA test in scenario %s fails in
%s%s


## Reproducible

Fails since (at least) Build %s


## Expected result

Last good: %s (or more recent)


## Further details

Always latest result in this scenario: [latest](%s)
""" % (scenario, urljoin(root_url, str(url)), details, first_known_bad, last_good, latest_link)

    config_section = 'product_issues:%s:product_mapping' % root_url.rstrip('/')
    # the test module name itself is often not specific enough, that is why we step upwards from the current module until we find the module folder and
    # concatenate the complete module name in format <folder>-<module> and search for that in the config for potential mappings
    first_step_url = urljoin(str(url), '1/src')
    start_of_current_module = test_details_page.find('a', {'href': first_step_url})
    try:
        module_folder = start_of_current_module.parent.parent.parent.parent.find(class_='glyphicon-folder-open').parent.text.strip()
    except AttributeError:  # pragma: no cover
        module_folder = ''
        log.warn("Could not find module folder on test details page searching for parents of %s" % first_step_url)
    complete_module = module_folder + '-' + module
    component_config_section = 'product_issues:%s:component_mapping' % root_url.rstrip('/')
    try:
        components_config_dict = dict(config.items(component_config_section))
        component = [v for k, v in iteritems(components_config_dict) if re.match(k, complete_module)][0]
    except (NoSectionError, IndexError) as e:  # pragma: no cover
        log.info("No matching component could be found for the module_folder '%s' and module name '%s' in the config section '%s'" % (module_folder, module, e))
        component = ''
    try:
        product = config.get(config_section, group)
    except NoOptionError as e:  # pragma: no cover
        log.info("%s. Reporting link for product will not work." % e)
        product = ''
    product_entries = OrderedDict([
        ('product', product),
        ('component', component),
        ('short_desc', '[Build %s] openQA test fails%s' % (build, ' in %s' % module if module else '')),
        ('bug_file_loc', urljoin(root_url, str(url))),
        ('comment', description)
    ])
    product_bug = urljoin(str(config.get('product_issues', 'report_url')), 'enter_bug.cgi') + '?' + urlencode(product_entries)
    test_entries = OrderedDict([
        ('issue[subject]', '[Build %s] test %sfails' % (build, module + ' ' if module else '')),
        ('issue[description]', description)
    ])
    test_issue = config.get('test_issues', 'report_url') + '?' + urlencode(test_entries)
    return ': report [product bug](%s) / [openQA issue](%s)' % (product_bug, test_issue)


class Issue(object):

    """Issue with extra status info from issue tracker."""

    def __init__(self, bugref, bugref_href, query_issue_status=False, progress_browser=None, bugzilla_browser=None):
        """Construct an issue object with options."""
        self.bugref = bugref
        self.bugref_href = bugref_href
        self.bugid = int(re.search('[a-zA-Z]*#([0-9]+)', bugref).group(1))
        self.msg = None
        self.json = None
        self.subject = None
        self.status = None
        self.assignee = None
        self.resolution = None
        self.priority = None
        self.queried = False
        self.last_comment_date = None
        self.issue_type = None
        self.error = False
        self.progress_browser = progress_browser
        self.bugzilla_browser = bugzilla_browser
        if query_issue_status and progress_browser and bugzilla_browser:
            log.debug("Retrieving bug data for %s" % bugref)
            try:
                if bugref.startswith('poo#'):
                    log.debug("Test issue discovered, looking on progress")
                    self.issue_type = 'redmine'
                    self.json = progress_browser.get_json(bugref_href + '.json')['issue']
                    self.status = self.json['status']['name']
                    self.assignee = self.json['assigned_to']['name'] if 'assigned_to' in self.json else 'None'
                    self.subject = self.json['subject']
                    self.priority = self.json['priority']['name']
                    self.last_comment_date = datetime.datetime.strptime(self.json['updated_on'], "%Y-%m-%dT%H:%M:%SZ")
                # bugref.startswith('bsc#') or bugref.startswith('boo#')
                else:
                    log.debug("Product bug discovered, looking on bugzilla")
                    self.issue_type = 'bugzilla'
                    self.json = bugzilla_browser.json_rpc_get('/jsonrpc.cgi', 'Bug.get', {"ids": [self.bugid]})['result']['bugs'][0]
                    self.status = self.json['status']
                    if self.json.get('resolution'):
                        self.resolution = self.json['resolution']
                    self.assignee = self.json['assigned_to'] if 'assigned_to' in self.json else 'None'
                    self.subject = self.json['summary']
                    self.priority = self.json['priority'].split(' ')[0] + '/' + self.json['severity']
                self.queried = True
            except DownloadError as e:  # pragma: no cover
                log.info("A download error has been encountered for bugref %s (%s): %s" % (bugref, bugref_href, e))
                self.msg = str(e)
                self.error = True
            except TypeError as e:
                log.error("Error retrieving details for bugref %s (%s): %s" % (bugref, bugref_href, e))
                self.msg = "Ticket not found"
                self.error = True

    def add_comment(self, comment):
        """Add a comment to an issue with RPC/REST operations."""
        log.info("Posting a comment on %s ticket [%s](%s)" % (self.issue_type, self.bugref, self.bugref_href))
        if self.issue_type == 'bugzilla':
            self.bugzilla_browser.json_rpc_post('/jsonrpc.cgi', 'Bug.add_comment', {
                "id": self.bugid,
                "comment": comment,
                "is_private": True,
            })
        # self.issue_type == 'redmine':
        else:
            self.progress_browser.json_rest(self.bugref_href + '.json', 'PUT', {
                "issue": {
                    "notes": comment
                }
            })

    @property
    def is_assigned(self):
        """Issue has been assigned."""
        assert self.queried
        if self.assignee == 'None':
            return False
        elif "@forge.provo.novell.com" in self.assignee:
            return False
        else:
            return True

    @property
    def is_open(self):
        """Issue is still open."""
        assert self.queried
        s = self.status.upper()
        if s in ["RESOLVED", "REJECTED", "VERIFIED", "CLOSED"]:
            return False
        else:
            return True

    @property
    def last_comment(self):
        """Return datetime object of last comment retrieved from an issue."""
        if not self.last_comment_date:
            assert self.issue_type == 'bugzilla'
            res = self.bugzilla_browser.json_rpc_get('/jsonrpc.cgi', 'Bug.comments', {"ids": [self.bugid]})
            comments = res['result']['bugs'][str(self.bugid)]['comments']
            self.last_comment_date = datetime.datetime.strptime(comments[-1]['creation_time'], "%Y-%m-%dT%H:%M:%SZ")
        return self.last_comment_date

    def __str__(self):
        """Format issue using markdown."""
        if self.msg:
            msg = self.msg
        elif self.status:
            status = self.status
            if self.resolution:
                status += ' (%s)' % self.resolution
            if status.startswith('VERIFIED') or status.startswith('Resolved'):
                status = '<span style="color: red;">%s</span>' % status
            msg = 'Ticket status: %s, prio/severity: %s, assignee: %s' % (status, self.priority, self.assignee)
        else:
            msg = None
        return '[%s](%s%s)%s' % (
            self.bugref,
            self.bugref_href,
            ' "%s"' % self.subject.replace(')', '&#41;') if self.subject else '',
            ' (%s)' % msg if msg else '',
        )


class IssueEntry(object):

    """List of failed test scenarios with corresponding bug."""

    def __init__(self, args, root_url, failures, test_browser=None, bug=None):
        """Construct an issueentry object with options."""
        self.args = args
        self.failures = [f for f in failures]
        self.bug = bug
        self.soft = self.failures[0]['state'] in soft_fail_states
        self.root_url = root_url
        self.test_browser = test_browser

    def _url(self, v):
        """Absolute url e.g. for test references."""
        return absolute_url(self.root_url, v)

    def _format_failure_modules(self, failedmodules):
        return ', '.join(m['name'] for m in failedmodules)

    def _format_failure(self, f):
        """Yield a report entry for one new issue based on verbosity."""
        failure_modules_str = ' "Failed modules: %s"' % self._format_failure_modules(f['failedmodules']) if f['failedmodules'] else ''
        report_str = issue_report_link(self.root_url, f, self.test_browser) if (self.args.report_links and self.test_browser) else ''
        if self.args.verbose_test >= 3 and 'prev' in f:
            return '[%s](%s%s) [(ref)](%s "Previous test")%s' % (
                f['name'], self._url(f),
                failure_modules_str,
                self._url(f['prev']),
                report_str
            )
        elif self.args.verbose_test >= 2:
            return '[%s](%s%s)%s' % (f['name'], self._url(f), failure_modules_str, report_str)
        else:
            return '%s' % f['name']

    def __str__(self):
        """Return as markdown."""
        test_bug_str = '%s%s' % (
            ', '.join(map(self._format_failure, self.failures)),
            ' -> %s' % self.bug if self.bug else ''
        )
        short_str = self.bug if self.bug else ', '.join(i['name'] for i in self.failures)
        return '* %s%s\n' % (
            'soft fails: ' if self.soft else '',
            short_str if self.args.short_failure_str else test_bug_str
        )

    @classmethod
    def for_each(cls, args, root_url, failures, test_browser):
        """Create one object for each failure (for todo entries)."""
        return map(lambda f: cls(args, root_url, [f], test_browser), failures)


class ArchReport(object):

    """Report for a single architecture."""

    def __init__(self, arch, results, args, root_url, progress_browser, bugzilla_browser, test_browser):
        """Construct an archreport object with options."""
        self.arch = arch
        self.args = args
        self.root_url = root_url
        self.progress_browser = progress_browser
        self.bugzilla_browser = bugzilla_browser
        self.test_browser = test_browser

        self.status_badge = set_status_badge([i['state'] for i in results.values()])

        if self.args.bugrefs and self.args.include_softfails:
            self._search_for_bugrefs_for_softfailures(results)

        # if a ticket is known and the same refers to a STILL_FAILING scenario and any NEW_ISSUE we regard that as STILL_FAILING but just visible in more
        # scenarios, ...
        # ... else (no ticket linked) we don't group them as we don't know if it really is the same issue and handle them outside
        results_by_bugref = SortedDict(get_results_by_bugref(results, self.args))
        self.issues = defaultdict(lambda: defaultdict(list))
        for bugref, result_list in iteritems(results_by_bugref):
            if not re.match('(poo|bsc|boo)#', bugref):
                log.info('Skipping unknown bugref \'%s\' in \'%s\'' % (bugref, result_list))
                continue
            bug = result_list[0]
            issue = Issue(bug['bugref'], bug['bugref_href'], self.args.query_issue_status, self.progress_browser, self.bugzilla_browser)
            self.issues[issue_state(result_list)][issue_type(bugref)].append(IssueEntry(self.args, self.root_url, result_list, bug=issue))

        # left to handle are the issues marked with 'todo'
        todo_results = results_by_bugref.get('todo', [])
        new_issues = (r for r in todo_results if r['state'] == 'NEW_ISSUE')
        self.issues['new']['todo'].extend(IssueEntry.for_each(self.args, self.root_url, new_issues, test_browser))
        existing_issues = (r for r in todo_results if r['state'] == 'STILL_FAILING')
        self.issues['existing']['todo'].extend(IssueEntry.for_each(self.args, self.root_url, existing_issues, test_browser))
        if self.args.include_softfails:
            new_soft_fails = [r for r in todo_results if r['state'] == 'NEW_SOFT_ISSUE']
            existing_soft_fails = [r for r in todo_results if r['state'] == 'STILL_SOFT_FAILING']
            if new_soft_fails:
                self.issues['new']['product'].append(IssueEntry(self.args, self.root_url, new_soft_fails))
            if existing_soft_fails:
                self.issues['existing']['product'].append(IssueEntry(self.args, self.root_url, existing_soft_fails))

    def _search_for_bugrefs_for_softfailures(self, results):
        for k, v in iteritems(results):
            if v['state'] in soft_fail_states:
                try:
                    module_url = self._get_url_to_softfailed_module(v['href'])
                    module_name = re.search("[^/]*/[0-9]*/[^/]*/([^/]*)/[^/]*/[0-9]*", module_url).group(1)
                    assert module_name, 'could not find a module name within %s in job %s' % (module_url, v['href'])
                    v['bugref'] = self._get_bugref_for_softfailed_module(v, module_name)
                    if not v['bugref']:  # pragma: no cover
                        continue
                except AttributeError:  # pragma: no cover
                    log.info('Could find neither soft failed info box nor needle, assuming an old openQA job, skipping.')
                    continue
                except DownloadError as e:  # pragma: no cover
                    log.error("Failed to process %s with error %s. Skipping current result" % (v, e))
                    continue
                match = re.search('([a-z]{3})#?([0-9]+)', v['bugref'])
                if not match:  # pragma: no cover
                    log.info('Could not find bug reference in text \'%s\', skipping.' % v['bugref'])
                    continue
                bugref, bug_id = match.group(1), match.group(2)
                assert bugref, "No bugref found for %s" % v
                assert bug_id, "No bug_id found for %s" % v
                try:
                    v['bugref_href'] = issue_tracker[bugref](bug_id)
                except KeyError as e:  # pragma: no cover
                    log.error("Failed to find valid bug tracker URL for %s with error %s. Skipping current result" % (v, e))
                    continue

    @property
    def total_issues(self):
        """Return Number of issue entries for this arch."""
        total = 0
        for issue_status, issue_types in iteritems(self.issues):
            for issue_type, ies in iteritems(issue_types):
                total += len(ies)
        return total

    def _get_url_to_softfailed_module(self, job_url):
        test_details_html = self.test_browser.get_soup(job_url).find(title="Soft Failed")
        if test_details_html is None:
            log.debug('Could not find soft failed info box, looking for workaround needle in job %s' % job_url)
            test_details_html = self.test_browser.get_soup(job_url).find(class_='resborder_softfailed').parent
        assert test_details_html, 'Found neither soft failed info box nor workaround needle'
        return test_details_html.get('data-url')

    def _get_bugref_for_softfailed_module(self, result_item, module_name):
        details_json = json.loads(self.test_browser.get_soup("%s/file/details-%s.json" % (result_item['href'], module_name)).getText())
        for field in details_json:
            if 'title' in field and 'Soft Fail' in field['title']:
                unformated_str = self.test_browser.get_soup("%s/file/%s" % (result_item['href'], field['text'])).getText()
                return re.search("Soft Failure:\n([^/]*)", unformated_str.strip()).group(1)
            elif 'properties' in field and len(field['properties']) > 0 and field['properties'][0] == 'workaround':
                log.debug('Evaluating potential workaround needle \'%s\'' % field['needle'])
                match = re.search('([a-z]{3})#?([0-9]+)-[0-9]+', field['needle'])
                if not match:  # pragma: no cover
                    log.warn('Found workaround needle without bugref that could be understood, looking for a better bugref (if any) for \'%s\'' %
                             result_item['href'])
                    continue
                return match.group(1) + '#' + match.group(2)
        else:  # pragma: no cover
            log.error('Could not find any soft failure reference within details of soft-failed job \'%s\'. Could be deleted workaround needle?.' %
                      absolute_url(self.root_url, result_item))

    def _todo_issues_str(self):
        if self.args.abbreviate_test_issues:
            return issue_listing('### Test issues', self.issues['new']['openqa'] + self.issues['existing']['openqa'] + self.issues['new']['todo'] +
                                 self.issues['existing']['todo'], self.args.show_empty)
        todo_issues = todo_review_template.substitute({
            'new_issues': issue_listing('***new issues***', self.issues['new']['todo'], self.args.show_empty),
            'existing_issues': issue_listing('***existing issues***', self.issues['existing']['todo'], self.args.show_empty),
        })
        return todo_issues if (self.issues['new']['todo'] or self.issues['existing']['todo']) else ''

    def __str__(self):
        """Return as markdown."""
        abbrev = self.args.abbreviate_test_issues
        return openqa_review_report_arch_template.substitute({
            'arch': self.arch,
            'status_badge': status_badge_str[self.status_badge],
            # everything that is 'NEW_ISSUE' should be product issue but if tests have changed content, then probably openqa issues
            # For now we can just not easily decide unless we use the 'bugrefs' mode
            'new_openqa_issues': '' if abbrev else issue_listing('**New openQA-issues:**', self.issues['new']['openqa'], self.args.show_empty),
            'existing_openqa_issues': '' if abbrev else issue_listing('**Existing openQA-issues:**', self.issues['existing']['openqa'], self.args.show_empty),
            'new_product_issues': issue_listing('**New Product bugs:**', self.issues['new']['product'], self.args.show_empty),
            'existing_product_issues': issue_listing('**Existing Product bugs:**', self.issues['existing']['product'], self.args.show_empty),
            'todo_issues': self._todo_issues_str(),
        })


class ProductReport(object):

    """Read overview page of one job group and generate a report for the product."""

    def __init__(self, browser, job_group_url, root_url, args):
        """Construct a product report object with options."""
        self.args = args
        self.job_group_url = job_group_url
        self.group = job_group_url.split('/')[-1]
        current_url, previous_url = get_build_urls_to_compare(browser, job_group_url, args.builds, args.against_reviewed, args.running_threshold)
        # read last finished
        current_details = browser.get_soup(current_url)
        previous_details = browser.get_soup(previous_url)
        for details in current_details, previous_details:
            assert sum(int(badge.text) for badge in details.find_all(class_='badge')) > 0, \
                "invalid page with no test results found reading %s and %s, make sure you specified valid builds (leading zero missing?)" \
                % (current_url, previous_url)
        current_summary = parse_summary(current_details)
        previous_summary = parse_summary(previous_details)

        changes = SortedDict({k: v - previous_summary.get(k, 0) for k, v in iteritems(current_summary) if k != 'none'})
        self.changes_str = '***Changes since reference build***\n\n* ' + '\n* '.join("%s: %s" % (k, v) for k, v in iteritems(changes)) + '\n'
        log.info("%s" % self.changes_str)

        self.build = get_build_nr(current_url)
        self.ref_build = get_build_nr(previous_url)

        # for each architecture iterate over all
        cur_archs, prev_archs = (set(arch.text for arch in details.find_all('th', id=re.compile('flavor_'))) for details in [current_details, previous_details])
        archs = cur_archs
        if args.arch:
            assert args.arch in cur_archs, "Selected arch {} was not found in test results {}".format(args.arch, cur_archs)
            archs = [args.arch]
        self.missing_archs = sorted(prev_archs - cur_archs)
        if self.missing_archs:
            log.info("%s missing completely from current run: %s" %
                     (pluralize(len(self.missing_archs), "architecture is", "architectures are"), ', '.join(self.missing_archs)))

        # create arch reports
        self.reports = SortedDict()
        progress_browser = progress_browser_factory(args) if args.query_issue_status else None
        bugzilla_browser = bugzilla_browser_factory(args) if args.query_issue_status else None
        for arch in sorted(archs):
            results = get_arch_state_results(arch, current_details, previous_details, args.output_state_results)
            self.reports[arch] = ArchReport(arch, results, args, root_url, progress_browser, bugzilla_browser, browser)

    def __str__(self):
        """Return report for product."""
        now_str = datetime.datetime.now().strftime('%Y-%m-%d - %H:%M')
        missing_archs_str = '\n * **Missing architectures**: %s' % ', '.join(self.missing_archs) if self.missing_archs else ''

        build_str = self.build
        if self.args.verbose_test and self.args.verbose_test > 1:
            build_str += ' (reference %s)' % self.ref_build
        if self.args.verbose_test and self.args.verbose_test > 3:
            build_str += '\n\n' + self.changes_str

        openqa_review_report_product = openqa_review_report_product_template.substitute({
            'now': now_str,
            'build': build_str,
            'common_issues': common_issues(missing_archs_str, self.args.show_empty),
            'arch_report': '\n---\n'.join(map(str, self.reports.values()))
        })
        return openqa_review_report_product


class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter):
    """Preserve multi-line __doc__ and provide default arguments in help strings."""
    pass


def parse_args():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=CustomFormatter)
    parser.add_argument('-v', '--verbose',
                        help="Increase verbosity level, specify multiple times to increase verbosity",
                        action='count', default=1)
    parser.add_argument('-n', '--no-progress', action='store_true',
                        help="Be terse and only output the report, no progress indication")
    parser.add_argument('-s', '--output-state-results', action='store_true',
                        help='Additional plain text output of arch-specific state results, e.g. all NEW_ISSUE; on for "verbose" mode')
    parser.add_argument('--host', default='https://openqa.opensuse.org',
                        help='openQA host to access')
    parser.add_argument('-j', '--job-groups',
                        help="""Only handle selected job group(s), comma separated, e.g. \'openSUSE Tumbleweed Gnome\'.
                        A regex also works, e.g. \'openSUSE Tumbleweed\' or \'(Gnome|KDE)\'.""")
    parser.add_argument('--exclude-job-groups',
                        help="""Exclude selected job groups by regex, inverse of '--job-groups'.""")
    parser.add_argument('-J', '--job-group-urls',
                        help="""Only handle selected job group(s) specified by URL, comma separated. Overwrites "--host" argument.
                        Skips parsing on main page and can actually save some seconds.""")
    builds = parser.add_mutually_exclusive_group()
    builds.add_argument('-b', '--builds',
                        help="""Select explicit builds, comma separated.
                        Specify as unambigous search terms, e.g. build number,
                        the full string, etc. Only works with single job-group/job-group-urls.
                        Default 'last' and 'previous'.""")
    builds.add_argument('-B', '--against-reviewed', metavar='BUILD',
                        help="""Compare specified build against last reviewed (as found in comments section).
                        E.g. if the last reviewed job was '0123' and you want to compare build '0128' against '0123',
                        specify just '0128' and the last reviewed job is found from the comments section if the comment
                        is sticking to the template format for review comments.
                        Special argument 'last' will compare the last finished build against the last reviewed one.""")
    parser.add_argument('-T', '--verbose-test',
                        help='Increase test result verbosity level, specify multiple times to increase verbosity',
                        action='count', default=1)
    parser.add_argument('-r', '--bugrefs', action='store_true',
                        help="""Parse \'bugrefs\' from test results comments and triage issues accordingly.
                        See https://progress.opensuse.org/projects/openqav3/wiki/Wiki#Show-bug-or-label-icon-on-overview-if-labeled-gh550
                        for details about bugrefs in openQA""")
    parser.add_argument('--short-failure-str', action='store_true',
                        help="""Instead of the default long failure description use a short version only outputting the referenced bug or the failing test in
                        case of failure without bugref.""")
    parser.add_argument('--abbreviate-test-issues', action='store_true',
                        help="""If requested abbreviate all 'test issue' related reporting sections and instead only show a short list of these next to the
                        full-detail product issues. Useful to focus on product-related reports.""")
    parser.add_argument('-R', '--query-issue-status', action='store_true',
                        help="""Query issue trackers for the issues found and report on their status and assignee. Implies "-r/--bugrefs" and
                        needs configuration file {} with credentials, see '--query-issue-status-help'.""".format(CONFIG_PATH))
    parser.add_argument('--query-issue-status-help', action='store_true',
                        help="""Shows help how to setup '--query-issue-status' configuration file.""")
    parser.add_argument('--report-links', action='store_true',
                        help="""Generate issue reporting links into report. Needs configuration file for product mapping,
                        see '--query-issue-status-help'.""")
    parser.add_argument('-a', '--arch',
                        help='Only single architecture, e.g. \'x86_64\', not all')
    parser.add_argument('-f', '--filter',
                        help='Filter for \'closed\' or \'unassigned\' issues.')
    parser.add_argument('--running-threshold', default=0,
                        help='Percentage of jobs that may still be running for the build to be considered \'finished\' anyway')
    parser.add_argument('--no-empty-sections', action='store_false', default=True, dest='show_empty',
                        help='Only show sections in report with content')
    parser.add_argument('--include-softfails', action='store_true', default=False,
                        help="""Also include softfails in reports.
                        Not included by default as less important and there is
                        no carry over for soft fails, i.e. there are no
                        bugrefs attached to these failures in most cases but
                        they should already carry bug references by other
                        means anyway.""")
    reminder_comments = parser.add_argument_group('Reminder comments on found issues')
    reminder_comments.add_argument('--reminder-comment-on-issues', action='store_true', default=False,
                                   help="""Go through bugrefs and write an actual comment on the ticket with a corresponding
                                   current job URL in the ticket if it has not been updated for some time.
                                   Implies '--query-issue-status'.""")
    reminder_comments.add_argument('--dry-run', action='store_true', default=False,
                                   help="""Do not actually change any tickets.""")
    reminder_comments.add_argument('--min-days-unchanged', default=MIN_DAYS_UNCHANGED,
                                   help="""The minimum period of days that need to be passed since the last comment for the bug to be reminded upon.""")
    add_load_save_args(parser)
    args = parser.parse_args()
    if args.query_issue_status_help:
        print(CONFIG_USAGE)
        print("Expected file path: {}".format(CONFIG_PATH))
        sys.exit(0)
    if args.reminder_comment_on_issues:
        args.query_issue_status = True
    if args.query_issue_status:
        args.bugrefs = True
    return args


def get_parent_job_groups(browser, root_url, args):
    pgroup_api_url = urljoin(root_url, 'api/v1/parent_groups')
    if args.no_progress or not humanfriendly_available:
        response = browser.get_json(pgroup_api_url)
    else:
        with AutomaticSpinner(label='Retrieving parent job groups'):
            response = browser.get_json(pgroup_api_url)
    return {p['id']: p['name'] for p in response}


def get_job_groups(browser, root_url, args):
    if args.job_group_urls:
        job_group_urls = args.job_group_urls.split(',')
        log.info("Acting on specified job group URL(s): %s" % ', '.join(job_group_urls))
        job_groups = {i: url for i, url in enumerate(job_group_urls)}
    else:
        parent_groups = get_parent_job_groups(browser, root_url, args)
        if args.no_progress or not humanfriendly_available:
            results = browser.get_json(urljoin(root_url, 'api/v1/job_groups'))
        else:
            with AutomaticSpinner(label='Retrieving job groups'):
                results = browser.get_json(urljoin(root_url, 'api/v1/job_groups'))

        def _pgroup_prefix(group):
            try:
                return '%s / %s' % (parent_groups[group['parent_id']], group['name'])
            except KeyError:
                return group['name']

        job_groups = {}
        for job_group in results:
            job_groups[_pgroup_prefix(job_group)] = urljoin(root_url, '/group_overview/%i' % job_group['id'])
        if args.job_groups:
            job_pattern = re.compile('(%s)' % '|'.join(args.job_groups.split(',')))
            job_groups = {k: v for k, v in iteritems(job_groups) if job_pattern.search(k)}
            log.info("Job group URL for %s: %s" % (args.job_groups, job_groups))
        if args.exclude_job_groups:
            job_pattern = re.compile('(%s)' % '|'.join(args.exclude_job_groups.split(',')))
            job_groups = {k: v for k, v in iteritems(job_groups) if not job_pattern.search(k)}
            log.info("Job group URL excluding %s: %s" % (args.exclude_job_groups, job_groups))
    return SortedDict(job_groups)


class Report(object):

    """openQA review report."""

    def __init__(self, browser, args, root_url, job_groups):
        """Create openQA review report."""
        self.browser = browser
        self.args = args
        self.root_url = root_url
        self.job_groups = job_groups

        self._label = 'Gathering data and processing report'
        self._progress = 0
        self.report = SortedDict()

        for k, v in iteritems(job_groups):
            log.info("Processing '%s'" % v)
            if args.no_progress or not humanfriendly_available:
                self.report[k] = self._one_report(v)
            else:
                with AutomaticSpinner(label=self._next_label()):
                    self.report[k] = self._one_report(v)
            self._progress += 1
        if not args.no_progress:
            sys.stderr.write("\r%s\n" % self._next_label())  # It's nice to see 100%, too :-)

    def _one_report(self, job_group_url):
        # for each job group on openqa.opensuse.org
        try:
            return ProductReport(self.browser, job_group_url, self.root_url, self.args)
        except NotEnoughBuildsError as e:
            log.debug("Catched 'not enough builds': %s" % e)
            return "Not enough finished builds found"

    def _next_label(self):
        return '%s %i%%' % (self._label, self._progress * 100 / len(self.job_groups.keys()))

    def __str__(self):
        """Generate markdown."""
        report_str = ""
        for k, v in iteritems(self.report):
            report_str += '# %s\n\n%s\n---\n' % (k, v)
        return report_str


def generate_report(args):
    verbose_to_log = {
        0: logging.CRITICAL,
        1: logging.ERROR,
        2: logging.WARN,
        3: logging.INFO,
        4: logging.DEBUG
    }
    logging_level = logging.DEBUG if args.verbose > 4 else verbose_to_log[args.verbose]
    log.setLevel(logging_level)
    log.debug("args: %s" % args)
    args.output_state_results = True if args.verbose > 1 else args.output_state_results

    if args.job_group_urls:
        root_url = urljoin('/'.join(args.job_group_urls.split("/")[0:3]), '/')
    else:
        root_url = urljoin(args.host, '/')

    browser = Browser(args, root_url)
    job_groups = get_job_groups(browser, root_url, args)
    assert not (args.builds and len(job_groups) > 1), "builds option and multiple job groups not supported"
    assert len(job_groups) > 0, "No job groups were found, maybe misspecified '--job-groups'?"

    return Report(browser, args, root_url, job_groups)


def load_config():
    global config
    config = ConfigParser()
    config_entries = config.read(CONFIG_PATH)
    if not config_entries:  # pragma: no cover
        print("Need configuration file '{}' for issue retrieval credentials".format(CONFIG_PATH))
        print(CONFIG_USAGE)
        sys.exit(1)


ie_filters = {
    "closed": lambda ie: ie.bug and ie.bug.queried and not ie.bug.is_open,
    "unassigned": lambda ie: ie.bug and ie.bug.queried and ie.bug.is_open and not ie.bug.is_assigned
}


def filter_report(report, iefilter):
    report.report = SortedDict({p: pr for p, pr in iteritems(report.report) if isinstance(pr, ProductReport)})
    for product, pr in iteritems(report.report):
        for arch, ar in iteritems(pr.reports):
            for issue_status, issue_types in iteritems(ar.issues):
                for issue_type, ies in iteritems(issue_types):
                    issue_types[issue_type] = [ie for ie in ies if iefilter(ie)]
        pr.reports = SortedDict({a: ar for a, ar in iteritems(pr.reports) if ar.total_issues > 0})
    report.report = SortedDict({p: pr for p, pr in iteritems(report.report) if pr.reports})


def reminder_comment_on_issue(ie, min_days_unchanged=MIN_DAYS_UNCHANGED):
    issue = ie.bug
    if issue.error:
        return
    if (datetime.datetime.utcnow() - issue.last_comment).days >= min_days_unchanged:
        f = ie.failures[0]
        comment = openqa_issue_comment.substitute({'name': f['name'], 'url': ie._url(f)}).strip()
        issue.add_comment(comment)


def reminder_comment_on_issues(report, min_days_unchanged=MIN_DAYS_UNCHANGED):
    processed_issues = set()
    report.report = SortedDict({p: pr for p, pr in iteritems(report.report) if isinstance(pr, ProductReport)})
    for product, pr in iteritems(report.report):
        for arch, ar in iteritems(pr.reports):
            for issue_status, issue_types in iteritems(ar.issues):
                for issue_type, ies in iteritems(issue_types):
                    for ie in ies:
                        issue = ie.bug
                        if issue:
                            bugref = issue.bugref.replace('bnc', 'bsc').replace('boo', 'bsc')
                            if bugref not in processed_issues:
                                try:
                                    reminder_comment_on_issue(ie, min_days_unchanged)
                                except HTTPError as e:  # pragma: no cover
                                    log.error("Encountered error trying to post a reminder comment on issue '%s': %s. Skipping." % (ie, e))
                                    continue
                                processed_issues.add(bugref)


def main():  # pragma: no cover, only interactive
    args = parse_args()
    if args.query_issue_status or args.report_links:
        load_config()
    report = generate_report(args)

    if args.reminder_comment_on_issues:
        reminder_comment_on_issues(report)

    if args.filter:
        try:
            filter_report(report, ie_filters[args.filter])
        except KeyError:
            print("No such filter '%s'" % args.filter)
            print("Available filters: %s" % ', '.join(ie_filters.keys()))
            sys.exit(1)

    print(report)


if __name__ == "__main__":
    main()
