# -*- coding: utf-8 -*-


import collections
import copy
import json
import os
import re
import subprocess
from collections import defaultdict, OrderedDict
from datetime import datetime
from functools import partial

from . import attr
from . import config
from . import constants
from . import datablocks
from . import markdown
from .DefaultOrderedDict import DefaultOrderedDict
from .htmlhelpers import *
from .messages import *
from .repository import *


class MetadataManager:
    @property
    def vshortname(self):
        if self.level:
            return "{0}-{1}".format(self.shortname, self.level)
        return self.shortname

    @property
    def displayVshortname(self):
        if self.level:
            return "{0}-{1}".format(self.displayShortname, self.level)
        return self.displayShortname

    def __init__(self):
        self.hasMetadata = False

        # required metadata
        self.abstract = []
        self.ED = None
        self.level = None
        self.shortname = None
        self.status = None
        self.rawStatus = None

        # optional metadata
        self.advisementClass = "advisement"
        self.assertionClass = "assertion"
        self.assumeExplicitFor = False
        self.atRisk = []
        self.audience = []
        self.blockElements = []
        self.boilerplate = config.BoolSet(default=True)
        self.canIUseURLs = []
        self.canonicalURL = None
        self.complainAbout = config.BoolSet()
        self.customTextMacros = []
        self.customWarningText = []
        self.customWarningTitle = None
        self.date = datetime.utcnow().date()
        self.deadline = None
        self.defaultHighlight = None
        self.defaultRefStatus = None
        self.displayShortname = None
        self.editors = []
        self.editorTerm = {"singular": "Editor", "plural": "Editors"}
        self.favicon = None
        self.forceCrossorigin = False
        self.group = None
        self.h1 = None
        self.ignoreCanIUseUrlFailure = []
        self.ignoredTerms = []
        self.ignoredVars = []
        self.includeCanIUsePanels = False
        self.indent = 4
        self.inferCSSDfns = False
        self.informativeClasses = []
        self.inlineGithubIssues = False
        self.inlineTagCommands = {}
        self.issueClass = "issue"
        self.issues = []
        self.issueTrackerTemplate = None
        self.lineNumbers = False
        self.linkDefaults = defaultdict(list)
        self.logo = ""
        self.mailingList = None
        self.mailingListArchives = None
        self.markupShorthands = config.BoolSet(["css", "dfn", "biblio", "markup", "idl", "algorithm"])
        self.maxToCDepth = float('inf')
        self.metadataInclude = config.BoolSet(default=True)
        self.metadataOrder = ["*", "!*"]
        self.noAbstract = False
        self.noEditor = False
        self.noteClass = "note"
        self.opaqueElements = ["pre", "xmp", "script", "style"]
        self.prepTR = False
        self.previousEditors = []
        self.previousVersions = []
        self.removeMultipleLinks = False
        self.repository = config.Nil()
        self.requiredIDs = []
        self.slimBuildArtifact = False
        self.statusText = []
        self.testSuite = None
        self.title = None
        self.toggleDiffs = False
        self.TR = None
        self.trackingVectorAltText = "(This is a tracking vector.)"
        self.trackingVectorClass = "tracking-vector"
        self.trackingVectorImage = None
        self.trackingVectorImageHeight = "64"
        self.trackingVectorImageWidth = "46"
        self.trackingVectorTitle = "There is a tracking vector here."
        self.translateIDs = defaultdict(list)
        self.translations = []
        self.useDfnPanels = True
        self.useIAutolinks = False
        self.versionHistory = []
        self.warning = None
        self.workStatus = None
        self.wptDisplay = "none"
        self.wptPathPrefix = None

        self.otherMetadata = DefaultOrderedDict(list)

        self.manuallySetKeys = set()

    def addData(self, key, val, lineNum=None):
        key = key.strip()
        if isinstance(val, str):
            if key in ["Abstract"]:
                val = val.strip("\n")
            else:
                val = val.strip()

        if key.startswith("!"):
            key = key[1:]
            self.otherMetadata[key].append(val)
            return

        if key not in ("ED", "TR", "URL"):
            key = key.title()

        if key not in knownKeys:
            die('Unknown metadata key "{0}". Prefix custom keys with "!".', key, lineNum=lineNum)
            return
        md = knownKeys[key]

        val = md.parse(key=key, val=val, lineNum=lineNum)

        self.addParsedData(key, val)

    def addParsedData(self, key, val):
        md = knownKeys[key]
        result = md.join(getattr(self, md.attrName), val)
        setattr(self, md.attrName, result)
        self.manuallySetKeys.add(key)

    def computeImplicitMetadata(self, doc):
        # Do some "computed metadata", based on the value of other metadata.
        # Only call this when you're sure all metadata sources are parsed.
        if self.group == "byos":
            self.boilerplate.default = False
        if not self.repository and doc:
            self.repository = getSpecRepository(doc)
        if self.repository.type == "github" and "feedback-header" in self.boilerplate and "repository-issue-tracking" in self.boilerplate:
            self.issues.append(("GitHub", self.repository.formatIssueUrl()))
        self.status = config.canonicalizeStatus(self.rawStatus, self.group)

        # for TR document, defaults to TR
        # fallbacks to ED
        if (self.canonicalURL == None or self.canonicalURL == "TR") and self.TR:
            self.canonicalURL = self.TR
        elif (self.canonicalURL == None or self.canonicalURL == "ED") and self.ED:
            self.canonicalURL = self.ED

        if self.displayShortname:
            self.shortname = self.displayShortname.lower()

    def validate(self):
        if self.group == "byos":
            return True

        if not self.hasMetadata:
            die("The document requires at least one <pre class=metadata> block.")
            return

        # { MetadataManager attr : metadata name (for printing) }
        requiredSingularKeys = {
            'status': 'Status',
            'shortname': 'Shortname',
            'title': 'Title'
        }
        recommendedSingularKeys = {}
        requiredMultiKeys = {}

        if self.status not in config.noEDStatuses:
            requiredSingularKeys['ED'] = 'ED'
        if self.status in config.deadlineStatuses:
            requiredSingularKeys['deadline'] = 'Deadline'
        if self.status in config.datedStatuses:
            recommendedSingularKeys['date'] = 'Date'
        if self.status in config.snapshotStatuses:
            requiredSingularKeys['TR'] = "TR"
        if self.status not in config.unlevelledStatuses:
            requiredSingularKeys['level'] = 'Level'
        if self.status not in config.shortToLongStatus:
            die("Unknown Status '{0}' used.", self.status)
        if not self.noEditor:
            requiredMultiKeys['editors'] = "Editor"
        if not self.noAbstract:
            requiredMultiKeys['abstract'] = "Abstract"
        if self.group and self.group.lower() == "csswg":
            requiredSingularKeys['workStatus'] = "Work Status"
        if self.group and self.group.lower() == "wg21":
            requiredSingularKeys['audience'] = "Audience"

        errors = []
        warnings = []
        for attr, name in requiredSingularKeys.items():
            if getattr(self, attr) is None:
                errors.append("    Missing a '{0}' entry.".format(name))
        for attr, name in recommendedSingularKeys.items():
            if getattr(self, attr) is None:
                warnings.append("    You probably want to provide a '{0}' entry.".format(name))
        for attr, name in requiredMultiKeys.items():
            if len(getattr(self, attr)) == 0:
                errors.append("    Must provide at least one '{0}' entry.".format(name))
        if warnings:
            warn("Some recommended metadata is missing:\n{0}", "\n".join(warnings))
        if errors:
            die("Not all required metadata was provided:\n{0}", "\n".join(errors))
            return

    def fillTextMacros(self, macros, doc):
        # Fills up a set of text macros based on metadata.
        if self.title:
            macros["title"] = self.title
            macros["spectitle"] = self.title
        if self.h1:
            macros["spectitle"] = self.h1
        macros["shortname"] = self.displayShortname
        if self.statusText:
            macros["statustext"] = "\n".join(markdown.parse(self.statusText, self.indent))
        else:
            macros["statustext"] = ""
        macros["level"] = str(self.level)
        macros["vshortname"] = self.displayVshortname
        if self.status == "FINDING" and self.group:
            macros["longstatus"] = "Finding of the {0}".format(self.group)
        elif self.status in config.shortToLongStatus:
            macros["longstatus"] = config.shortToLongStatus[self.status]
        else:
            macros["longstatus"] = ""
        if self.status in ("w3c/LCWD", "w3c/FPWD"):
            macros["status"] = "WD"
        elif self.status in ("w3c/WG-NOTE", "w3c/IG-NOTE"):
            macros["status"] = "NOTE"
        else:
            macros["status"] = self.rawStatus
        if self.workStatus:
            macros["workstatus"] = self.workStatus
        if self.TR:
            macros["latest"] = self.TR
        if self.abstract:
            abstractLines = datablocks.transformDataBlocks(doc, self.abstract)
            macros["abstract"] = "\n".join(markdown.parse(abstractLines, self.indent))
            macros["abstractattr"] = escapeAttr("  ".join(abstractLines).replace("<<","<").replace(">>",">"))
        elif self.noAbstract:
            macros["abstract"] = ""
            macros["abstractattr"] = ""
        macros["year"] = self.date.year
        macros["date"] = self.date.strftime("{0} %B %Y".format(self.date.day))
        macros["date-dmmy"] = self.date.strftime("{0} %B %Y".format(self.date.day)) #same as plain 'date'
        macros["cdate"] = self.date.strftime("%Y%m%d")
        macros["isodate"] = self.date.strftime("%Y-%m-%d")
        macros["date-my"] = self.date.strftime("%b %Y")
        macros["date-mmy"] = self.date.strftime("%B %Y")
        if self.deadline:
            macros["deadline"] = self.deadline.strftime("{0} %B %Y".format(self.deadline.day))
            macros["isodeadline"] = self.deadline.strftime("%Y-%m-%d")
        if self.status in config.snapshotStatuses:
            macros["version"] = "https://www.w3.org/TR/{year}/{status}-{vshortname}-{cdate}/".format(**macros)
        elif self.ED:
            macros["version"] = self.ED
        macros["annotations"] = constants.testAnnotationURL
        if doc and self.vshortname in doc.testSuites:
            macros["testsuite"] = doc.testSuites[self.vshortname]['vshortname']
        if self.warning and len(self.warning) >= 2:
            macros["replacedby"] = self.warning[1]
        if self.warning and len(self.warning) >= 3:
            macros["snapshotid"] = self.warning[2]
        if self.warning and len(self.warning) >= 4:
            macros["snapshoturl"] = self.warning[3]
        macros["logo"] = self.logo
        if self.repository:
            macros["repository"] = self.repository.name
            macros["repositoryurl"] = self.repository.url
        if self.mailingList:
            macros["mailinglist"] = self.mailingList
        if self.mailingListArchives:
            macros["mailinglistarchives"] = self.mailingListArchives
        if self.status == "w3c/FPWD":
            macros["w3c-stylesheet-url"] = "https://www.w3.org/StyleSheets/TR/2016/W3C-WD"
        elif self.status == "FINDING":
            macros["w3c-stylesheet-url"] = "https://www.w3.org/StyleSheets/TR/2016/W3C-NOTE"
        elif self.status == "w3c/CG-DRAFT":
            macros["w3c-stylesheet-url"] = "https://www.w3.org/StyleSheets/TR/2016/cg-draft"
        elif self.status == "w3c/CG-FINAL":
            macros["w3c-stylesheet-url"] = "https://www.w3.org/StyleSheets/TR/2016/cg-final"
        else:
            shortStatus = self.rawStatus.partition("/")[2] if (self.rawStatus and "/" in self.rawStatus) else self.rawStatus
            macros["w3c-stylesheet-url"] = "https://www.w3.org/StyleSheets/TR/2016/W3C-{0}".format(shortStatus)
        if self.customWarningText is not None:
            macros["customwarningtext"] = "\n".join(markdown.parse(self.customWarningText, self.indent))
        if self.customWarningTitle is not None:
            macros["customwarningtitle"] = self.customWarningTitle
        # Custom macros
        for name, text in self.customTextMacros:
            macros[name.lower()] = text


def parseDate(key, val, lineNum):
    if val == "now":
        return datetime.utcnow().date()
    try:
        return datetime.strptime(val, "%Y-%m-%d").date()
    except:
        die("The {0} field must be in the format YYYY-MM-DD - got \"{1}\" instead.", key, val, lineNum=lineNum)
        return None


def parseLevel(key, val, lineNum):
    val = val.lower().strip()
    if val == "none":
        return ""
    return val.strip()


def parseInteger(key, val, lineNum):
    return int(val)


def parseBoolean(key, val, lineNum):
    b = boolish(val)
    if b is None:
        die("The {0} field must be true/false, yes/no, y/n, or on/off. Got '{1}' instead.", key, val, lineNum=lineNum)
    return b


def boolish(val):
    if val.lower() in ("true", "yes", "y", "on"):
        return True
    if val.lower() in ("false", "no", "n", "off"):
        return False
    return None


def parseWarning(key, val, lineNum):
    if val.lower() == "obsolete":
        return "warning-obsolete",
    if val.lower() == "not ready":
        return "warning-not-ready",
    if val.lower() == "custom":
        return "warning-custom",
    match = re.match(r"Commit +([^ ]+) +(.+) +replaced by +(.+)", val, re.I)
    if match:
        return "warning-commit", match.group(3), match.group(1), match.group(2)
    match = re.match(r"Branch +([^ ]+) +(.+) +replaced by +(.+)", val, re.I)
    if match:
        return "warning-branch", match.group(3), match.group(1), match.group(2)
    match = re.match(r"Replaced By +(.+)", val, re.I)
    if match:
        return "warning-replaced-by", match.group(1)
    match = re.match(r"New Version +(.+)", val, re.I)
    if match:
        return "warning-new-version", match.group(1)
    die('''Unknown value "{1}" for "{0}" metadata. Expected one of:
  obsolete
  not ready
  replaced by [new url]
  new version [new url]
  commit [snapshot id] [snapshot url] replaced by [master url]
  branch [branch name] [branch url] replaced by [master url]
  custom''', key, val, lineNum=lineNum)
    return None


def parseEditor(key, val, lineNum):
    pieces = [unescape(piece.strip()) for piece in val.split(',')]

    def looksLinkish(string):
        return re.match(r"\w+:", string) or looksEmailish(string)

    def looksEmailish(string):
        return re.match(r".+@.+\..+", string)
    data = {
        'name'   : pieces[0],
        'w3cid'  : None,
        'org'    : None,
        'orglink': None,
        'link'   : None,
        'email'  : None,
        'w3cid'  : None
    }
    # Handle well-known pieces, split off the ambiguous ones
    ambiguousPieces = []
    for piece in pieces[1:]:
        if re.match(r"w3cid \d+$", piece) and data['w3cid'] is None:
            data['w3cid'] = piece[6:]
        else:
            ambiguousPieces.append(piece)
    if len(ambiguousPieces) == 3 and looksLinkish(ambiguousPieces[1]) and looksLinkish(ambiguousPieces[2]):
        # [org, email, url] or [org, url, email]
        data['org'] = ambiguousPieces[0]
        if looksEmailish(ambiguousPieces[1]):
            data['email'] = ambiguousPieces[1]
            data['link'] = ambiguousPieces[2]
        else:
            data['link'] = ambiguousPieces[1]
            data['email'] = ambiguousPieces[2]
    elif len(ambiguousPieces) == 2 and looksLinkish(ambiguousPieces[0]) and looksLinkish(ambiguousPieces[1]):
        # [email, url] or [url, email]
        if looksEmailish(ambiguousPieces[0]):
            data['email'] = ambiguousPieces[0]
            data['link'] = ambiguousPieces[1]
        else:
            data['link'] = ambiguousPieces[0]
            data['email'] = ambiguousPieces[1]
    elif len(ambiguousPieces) == 2 and looksLinkish(ambiguousPieces[1]):
        # [org, email] or [org, url]
        data['org'] = ambiguousPieces[0]
        if looksEmailish(ambiguousPieces[1]):
            data['email'] = ambiguousPieces[1]
        else:
            data['link'] = ambiguousPieces[1]
    elif len(ambiguousPieces) == 1:
        # [org], [email], or [url]
        if looksLinkish(ambiguousPieces[0]):
            if looksEmailish(ambiguousPieces[0]):
                data['email'] = ambiguousPieces[0]
            else:
                data['link'] = ambiguousPieces[0]
        else:
            data['org'] = ambiguousPieces[0]
    elif len(ambiguousPieces) == 0:
        pass
    else:
        die("'{0}' format is '<name>, <company>?, <email-or-contact-page>?. Got:\n{1}", key, val, lineNum=lineNum)
        return []
    # Check if the org ends with a link
    if data['org'] is not None and " " in data['org'] and looksLinkish(data['org'].split()[-1]):
        pieces = data['org'].split()
        data['orglink'] = pieces[-1]
        data['org'] = ' '.join(pieces[:-1])
    # Check if the name ends with an ID.
    # TODO: remove this, it's redundant with the "w3cid ####" piece
    if data['name'] and re.search(r"\s\d+$", data['name']):
        pieces = data['name'].split()
        data['w3cid'] = pieces[-1]
        data['name'] = ' '.join(pieces[:-1])
    return [data]


def parseCommaSeparated(key, val, lineNum):
    return [term.strip().lower() for term in val.split(',')]


def parseLinkDefaults(key, val, lineNum):
    defaultSpecs = defaultdict(list)
    for default in val.split(","):
        match = re.match(r"^([\w\d-]+)  (?:\s+\( ({0}) (?:\s+(snapshot|current))? \) )  \s+(.*)$".format("|".join(config.dfnTypes.union(["dfn"]))), default.strip(), re.X)
        if match:
            spec = match.group(1)
            type = match.group(2)
            status = match.group(3)
            terms = match.group(4).split('/')
            dfnFor = None
            for term in terms:
                defaultSpecs[term.strip()].append((spec, type, status, dfnFor))
        else:
            die("'{0}' is a comma-separated list of '<spec> (<dfn-type>) <terms>'. Got:\n{1}", key, default, lineNum=lineNum)
            continue
    return defaultSpecs


def parseBoilerplate(key, val, lineNum):
    boilerplate = config.BoolSet(default=True)
    for command in val.split(","):
        pieces = command.lower().strip().split()
        if len(pieces) != 2:
            die("Boilerplate metadata pieces are a boilerplate label and a boolean. Got:\n{0}", command, lineNum=lineNum)
            continue
        if pieces[0] == "omit":
            # legacy syntax; allows "omit foo" in addition to the normal "foo off"/etc
            boilerplate[pieces[1]] = False
        else:
            onoff = boolish(pieces[1])
            if onoff is None:
                die("Boilerplate metadata pieces are a boilerplate label and a boolean. Got:\n{0}", command, lineNum=lineNum)
                continue
            boilerplate[pieces[0]] = onoff
    return boilerplate


def parseRefStatus(key, val, lineNum):
    val = val.strip().lower()
    if val == "dated":
        # Legacy term that used to be allowed
        val == "snapshot"
    if val in constants.refStatus:
        return val
    else:
        die("'{0}' must be either 'current' or 'snapshot'. Got '{1}'", key, val, lineNum=lineNum)
        return constants.refStatus.current


def parseComplainAbout(key, val, lineNum):
    validLabels = frozenset(["missing-example-ids", "broken-links", "accidental-2119", "missing-exposed"])
    ret = parseBoolishList(key, val.lower(), default=False, validLabels=validLabels, lineNum=lineNum)
    return ret


def parseBoolishList(key, val, default=None, validLabels=None, extraValues=None, lineNum=None):
    # Parses anything defined as "label <boolish>, label <boolish>" into a passed BoolSet
    # Supply a list of valid labels if you want to have them checked,
    # and a dict of {value=>bool} pairs you want in addition to the standard boolish values
    if default is None:
        boolset = {}
    elif default in (True, False):
        boolset = config.BoolSet(default=default)
    else:
        die("Programming error - parseBoolishList() got a non-bool default value: '{0}'", default)
    if extraValues is None:
        extraValues = {}
    vals = [v.strip() for v in val.split(",")]
    for v in vals:
        name,_, boolstring = v.strip().rpartition(" ")
        if not name or not boolstring:
            die("{0} metadata pieces are a label and a boolean. Got:\n{1}", key, v, lineNum=lineNum)
            continue
        if validLabels and name not in validLabels:
            die("Unknown {0} label '{1}'.", key, name, lineNum=lineNum)
            continue
        if boolstring in extraValues:
            boolset[name] = extraValues[boolstring]
        else:
            onoff = boolish(boolstring)
            if isinstance(onoff, bool):
                boolset[name] = onoff
            else:
                die("{0} metadata pieces are a shorthand category and a boolean. Got:\n{1}", key, v, lineNum=lineNum)
                continue
    return boolset


def parseLinkedText(key, val, lineNum):
    # Parses anything defined as "text url, text url, text url" into a list of 2-tuples.
    entries = []
    vals = [v.strip() for v in val.split(",")]
    for v in vals:
        entries.append(v.rsplit(" ", 1))
    return entries


def parseMarkupShorthands(key, val, lineNum):
    # Format is comma-separated list of shorthand category followed by boolean.
    # Output is a dict of the shorthand categories with boolean values.
    vals = [v.strip() for v in val.lower().split(",")]
    ret = config.BoolSet(default=False)
    validCategories = frozenset(["css", "markup", "dfn", "biblio", "idl", "markdown", "algorithm"])
    for v in vals:
        pieces = v.split()
        if len(pieces) != 2:
            die("Markup Shorthand metadata pieces are a shorthand category and a boolean. Got:\n{0}", v, lineNum=lineNum)
            continue
        name, boolstring = pieces
        if name not in validCategories:
            die("Unknown Markup Shorthand category '{0}'.", name, lineNum=lineNum)
            continue
        onoff = boolish(boolstring)
        if onoff is None:
            die("Markup Shorthand metadata pieces are a shorthand category and a boolean. Got:\n{0}", v, lineNum=lineNum)
            continue
        ret[name] = onoff
    return ret

def parseInlineGithubIssues(key, val, lineNum):
    val = val.lower()
    if val in ["title", "full"]:
        return val
    b = boolish(val)
    if b is None:
        die("Inline Github Issues must be 'title', 'full' or a boolish value. Got: '{0}'", val, lineNum=lineNum)
        return False
    elif b is True:
        return "full"
    else:
        return False

def parseTextMacro(key, val, lineNum):
    # Each Text Macro line is just a macro name (must be uppercase)
    # followed by the text it expands to.
    try:
        name, text = val.lstrip().split(None, 1)
    except:
        die("Text Macro lines must contain a macro name followed by the macro text. Got:\n{0}", val, lineNum=lineNum)
        return []
    if not re.match(r"[A-Z0-9-]+$", name):
        die("Text Macro names must be all-caps and alphanumeric. Got '{0}'", name, lineNum=lineNum)
        return []
    return [(name, text)]


def parseWorkStatus(key, val, lineNum):
    # The Work Status is one of (completed, stable, testing, refining, revising, exploring, rewriting, abandoned).
    val = val.strip().lower()
    if val not in ('completed', 'stable', 'testing', 'refining', 'revising', 'exploring', 'rewriting', 'abandoned'):
        die("Work Status must be one of (completed, stable, testing, refining, revising, exploring, rewriting, abandoned). Got '{0}'. See http://fantasai.inkedblade.net/weblog/2011/inside-csswg/process for details.", val, lineNum=lineNum)
        return None
    return val


def parseRepository(key, val, lineNum):
    # Shortname followed by url, or just url.
    # If just url, I'll try to recognize the shortname from it; otherwise it's the url again.
    val = val.strip()
    pieces = val.split(None, 1)
    if len(pieces) == 2:
        return Repository(url=pieces[0], name=pieces[1])
    elif len(pieces) == 1:
        # Try to recognize a GitHub url
        match = re.match("https://github\.([\w.-]+)/([\w-]+)/([\w-]+)/?$", val)
        if match:
            return GithubRepository(*match.groups())
        # If you just provide a user/repo pair, assume it's a github.com repo.
        # Will provide ways to opt into other repos when people care.
        match = re.match("([\w-]+)/([\w-]+)$", val)
        if match:
            return GithubRepository("com", *match.groups())
        # Otherwise just use the url as the shortname
        return Repository(url=val)
    else:
        die("Repository must be a url, optionally followed by a shortname. Got '{0}'", val, lineNum=lineNum)
        return config.Nil()


def parseTranslateIDs(key, val, lineNum):
    translations = {}
    for v in val.split(","):
        pieces = v.strip().split()
        if len(pieces) != 2:
            die("‘Translate IDs’ values must be an old ID followed by a new ID. Got '{0}'", v, lineNum=lineNum)
            continue
        old,new = pieces
        translations[old] = new
    return translations


def parseTranslation(key, val, lineNum):
    # Format is <lang-code> <url> [ [ , name <name-in-spec-lang> ] || [ , native-name <name-in-the-lang> ] ]?
    pieces = val.split(",")
    if not(1 <= len(pieces) <= 3):
        die("Format of a Translation line is <lang-code> <url> [ [ , name <name-in-spec-lang> ] || [ , native-name <name-in-the-lang> ] ]?. Got:\n{0}", val, lineNum=lineNum)
        return
    firstParts = pieces[0].split()
    if len(firstParts) != 2:
        die("First part of a Translation line must be a lang-code followed by a url. Got:\n{0}", pieces[0], lineNum=lineNum)
        return
    langCode, url = firstParts
    name = None
    nativeName = None
    for piece in pieces[1:]:
        k,v = piece.split(None, 1)
        if k.lower() == "name":
            name = v
        elif k.lower() == "native-name":
            nativeName = v
        else:
            die("Later parts of a Translation line must start with 'name' or 'native-name'. Got:\n{0}", piece, lineNum=lineNum)
    return [{"lang-code": langCode, "url": url, "name": name, "native-name": nativeName}]


def parseAudience(key, val, lineNum):
    # WG21 value
    values = [x.strip().upper() for x in val.strip().split(",")]
    if not values:
        die("Audience metadata must have at least one value if specified.")
        return []
    elif len(values) == 1 and values[0] == "ALL":
        return ["all"]
    elif len(values) >= 1:
        ret = []
        namedAudiences = {"CWG", "LWG", "EWG", "LEWG", "DIRECTION"}
        pseudonymAudiences = {"Concurrency":"SG1", "TM":"SG5", "Numerics":"SG6", "Reflection":"SG7", "UB":"SG12", "HMI":"SG13", "Tooling":"SG15", "Unicode":"SG16", "EWGI":"SG17", "LEWG":"SG18"}
        for v in values:
            if v in namedAudiences:
                ret.append(v)
            elif v in pseudonymAudiences:
                ret.append(pseudonymAudiences[v])
            elif re.match(r"WG\d+|SG\d+", v):
                ret.append(v)
            else:
                die("Unknown 'Audience' value '{0}'.", v, lineNum=lineNum)
                continue
        return ret


def parseEditorTerm(key, val, lineNum):
    values = [x.strip() for x in val.strip().split(",")]
    if len(values) == 2:
        return {"singular": values[0], "plural": values[1]}
    else:
        die("Editor Term metadata must be two comma-separated terms, giving the singular and plural term for editors. Got '{0}'.", val)
        return {"singular": "Editor", "plural": "Editors"}


def parseMaxToCDepth(key, val, lineNum):
    if val.lower() == "none":
        return float('inf')
    try:
        v = int(val)
    except ValueError as e:
        die("Max ToC Depth metadata must be 'none' or an integer 1-5. Got '{0}'.", val, lineNum=lineNum)
        return float('inf')
    if not (1 <= v <= 5):
        die("Max ToC Depth metadata must be 'none' or an integer 1-5. Got '{0}'.", val, lineNum=lineNum)
        return float('inf')
    return v


def parseMetadataOrder(key, val, lineNum):
    pieces = [x.strip() for x in val.split(",")]
    return pieces


def parseWptDisplay(key, val, lineNum):
    val = val.lower()
    if val in ("none", "inline"):
        return val
    die("WPT Display metadata only accepts the values 'none' or 'inline'. Got '{0}'.", val, lineNum=lineNum)
    return "none"


def parseInlineTagCommand(key, val, lineNum):
    tag,_,command = val.strip().partition(" ")
    command = command.strip()
    return {tag: command}


def parse(lines):
    # Given HTML document text, in the form of an array of text lines,
    # extracts all <pre class=metadata> lines and parses their contents.
    # Returns the text lines, with the metadata-related lines removed,
    # and a filled MetadataManager object

    newlines = []
    inMetadata = False
    lastKey = None
    endTag = None
    md = MetadataManager()
    for line in lines:
        if not inMetadata and re.match(r"<(pre|xmp) [^>]*class=[^>]*metadata[^>]*>", line.text):
            inMetadata = True
            md.hasMetadata = True
            if line.text.startswith("<pre"):
                endTag = r"</pre>\s*"
            else:
                endTag = r"</xmp>\s*"
            continue
        elif inMetadata and re.match(endTag, line.text):
            inMetadata = False
            continue
        elif inMetadata:
            if lastKey and (line.text.strip() == "" or re.match(r"\s+", line.text)):
                # empty lines, or lines that start with 1+ spaces, continue previous key
                md.addData(lastKey, line.text, lineNum=line.i)
            elif re.match(r"([^:]+):\s*(.*)", line.text):
                match = re.match(r"([^:]+):\s*(.*)", line.text)
                md.addData(match.group(1), match.group(2), lineNum=line.i)
                lastKey = match.group(1)
            else:
                die("Incorrectly formatted metadata line:\n{0}", line.text, lineNum=line.i)
                continue
        elif re.match(r"\s*<h1[^>]*>.*?</h1>", line.text):
            if md.title is None:
                title = re.match(r"\s*<h1[^>]*>(.*?)</h1>", line.text).group(1)
                md.addData("Title", title, lineNum=line.i)
            newlines.append(line)
        else:
            newlines.append(line)
    return newlines, md


def fromCommandLine(overrides):
    # Given a list of strings representing command-line arguments,
    # finds the args that correspond to metadata keys
    # and fills a MetadataManager accordingly.
    md = MetadataManager()
    for o in overrides:
        match = re.match("--md-([^ =]+)=(.+)", o)
        if not match:
            # Not a metadata key
            continue
        # Convert the key into a metadata name
        key = match.group(1).replace("-", " ")
        if key not in ("ED", "TR"):
            key = key.title()
        val = match.group(2).strip()
        md.addData(key, val)
    return md


def fromJson(data, source=""):
    md = MetadataManager()
    try:
        defaults = json.loads(data, object_pairs_hook=OrderedDict)
    except Exception as e:
        if data != "":
            if source == "computed-metadata":
                die("Error loading computed-metadata JSON.\nCheck if you need to JSON-escape some characters in a text macro?\n{0}", str(e))
            else:
                die("Error loading {1} JSON:\n{0}", str(e), source)
        return md
    for key,val in defaults.items():
        if isinstance(val, str):
            md.addData(key, val)
        elif isinstance(val, list):
            for indivVal in val:
                md.addData(key, indivVal)
        else:
            die("JSON metadata values must be strings or arrays of strings. '{0}' is something else.", key)
            return md
    return md


def getSpecRepository(doc):
    '''
    Attempts to find the name of the repository the spec is a part of.
    Currently only searches for GitHub repos.
    Returns a "shortname" of the repo, and the full url.
    '''
    if doc and doc.inputSource and doc.inputSource != "-":
        source_dir = config.docPath(doc)
        old_dir = os.getcwd()
        try:
            os.chdir(source_dir)
            with open(os.devnull, "wb") as fnull:
                remotes = str(subprocess.check_output(["git", "remote", "-v"], stderr=fnull))
            os.chdir(old_dir)
            searches = [
              r"origin\tgit@github\.([\w.-]+):([\w-]+)/([\w-]+)\.git \(\w+\)",
              r"origin\thttps://github\.([\w.-]+)/([\w-]+)/([\w-]+)\.git \(\w+\)",
              r"origin\thttps://github\.([\w.-]+)/([\w-]+)/([\w-]+)/? \(\w+\)",
            ]
            for search_re in searches:
                search = re.search(search_re, remotes)
                if search:
                    return GithubRepository(*search.groups())
            return config.Nil()
        except subprocess.CalledProcessError:
            # check_output will throw CalledProcessError when not in a git repo
            os.chdir(old_dir)
            return config.Nil()


def parseDoc(doc):
    # Look through the doc for any additional metadata information that might be needed.

    for el in findAll(".replace-with-note-class", doc):
        removeClass(el, "replace-with-note-class")
        addClass(el, doc.md.noteClass)
    for el in findAll(".replace-with-issue-class", doc):
        removeClass(el, "replace-with-issue-class")
        addClass(el, doc.md.issueClass)
    for el in findAll(".replace-with-assertion-class", doc):
        removeClass(el, "replace-with-assertion-class")
        addClass(el, doc.md.assertionClass)
    for el in findAll(".replace-with-advisement-class", doc):
        removeClass(el, "replace-with-advisement-class")
        addClass(el, doc.md.advisementClass)

    if "feedback-header" in doc.md.boilerplate and "issues-index" in doc.md.boilerplate and find("." + doc.md.issueClass, doc) is not None:
            # There's at least one inline issue.
            doc.md.issues.append(("Inline In Spec", "#issues-index"))


def join(*sources):
    '''
    MetadataManager is a monoid
    '''
    md = MetadataManager()
    md.hasMetadata = any(x.hasMetadata for x in sources)
    for mdsource in sources:
        for k in mdsource.manuallySetKeys:
            mdentry = knownKeys[k]
            md.addParsedData(k, getattr(mdsource, mdentry.attrName))
        for k,v in mdsource.otherMetadata.items():
            md.otherMetadata[k].extend(v)
    return md

@attr.s(slots=True, frozen=True)
class Metadata(object):
    humanName = attr.ib()
    attrName = attr.ib()
    join = attr.ib()
    parse = attr.ib()


def joinValue(a, b):
    return b


def joinList(a, b):
    return a + b


def joinDict(a,b):
    x = {}
    x.update(a)
    x.update(b)
    return x


def joinBoolSet(a,b):
    x = copy.deepcopy(a)
    x.update(b)
    return x


def joinDdList(a,b):
    x = defaultdict(list)
    x.update(a)
    x.update(b)
    return x


def parseLiteral(key, val, lineNum):
    return val


def parseLiteralOrNone(key, val, lineNum):
    if val.lower() == "none":
        return None
    return val


def parseLiteralCaseless(key, val, lineNum):
    return val.lower()


def parseLiteralList(key, val, lineNum):
    return [val]


knownKeys = {
    "Abstract": Metadata("Abstract", "abstract", joinList, parseLiteralList),
    "Advisement Class": Metadata("Advisement Class", "advisementClass", joinValue, parseLiteral),
    "Assertion Class": Metadata("Assertion Class", "assertionClass", joinValue, parseLiteral),
    "Assume Explicit For": Metadata("Assume Explicit For", "assumeExplicitFor", joinValue, parseBoolean),
    "At Risk": Metadata("At Risk", "atRisk", joinList, parseLiteralList),
    "Audience": Metadata("Audience", "audience", joinList, parseAudience),
    "Block Elements": Metadata("Block Elements", "blockElements", joinList, parseCommaSeparated),
    "Boilerplate": Metadata("Boilerplate", "boilerplate", joinBoolSet, parseBoilerplate),
    "Can I Use Url": Metadata("Can I Use URL", "canIUseURLs", joinList, parseLiteralList),
    "Canonical Url": Metadata("Canonical URL", "canonicalURL", joinValue, parseLiteral),
    "Complain About": Metadata("Complain About", "complainAbout", joinBoolSet, parseComplainAbout),
    "Custom Warning Text": Metadata("Custom Warning Text", "customWarningText", joinList, parseLiteralList),
    "Custom Warning Title": Metadata("Custom Warning Title", "customWarningTitle", joinValue, parseLiteral),
    "Date": Metadata("Date", "date", joinValue, parseDate),
    "Deadline": Metadata("Deadline", "deadline", joinValue, parseDate),
    "Default Biblio Status": Metadata("Default Biblio Status", "defaultRefStatus", joinValue, parseRefStatus), #synonym of "Default Ref Status"
    "Default Highlight": Metadata("Default Highlight", "defaultHighlight", joinValue, parseLiteral),
    "Default Ref Status": Metadata("Default Ref Status", "defaultRefStatus", joinValue, parseRefStatus),
    "ED": Metadata("ED", "ED", joinValue, parseLiteral),
    "Editor": Metadata("Editor", "editors", joinList, parseEditor),
    "Editor Term": Metadata("Editor Term", "editorTerm", joinValue, parseEditorTerm),
    "Favicon": Metadata("Favicon", "favicon", joinValue, parseLiteral),
    "Force Crossorigin": Metadata("Force Crossorigin", "forceCrossorigin", joinValue, parseBoolean),
    "Former Editor": Metadata("Former Editor", "previousEditors", joinList, parseEditor),
    "Group": Metadata("Group", "group", joinValue, parseLiteral),
    "H1": Metadata("H1", "h1", joinValue, parseLiteral),
    "Ignore Can I Use Url Failure": Metadata("Ignore Can I Use Url Failure", "ignoreCanIUseUrlFailure", joinList, parseLiteralList),
    "Ignored Terms": Metadata("Ignored Terms", "ignoredTerms", joinList, parseCommaSeparated),
    "Ignored Vars": Metadata("Ignored Vars", "ignoredVars", joinList, parseCommaSeparated),
    "Include Can I Use Panels": Metadata("Include Can I Use Panels", "includeCanIUsePanels", joinValue, parseBoolean),
    "Indent": Metadata("Indent", "indent", joinValue, parseInteger),
    "Infer Css Dfns": Metadata("Infer Css Dfns", "inferCSSDfns", joinValue, parseBoolean),
    "Informative Classes": Metadata("Informative Classes", "informativeClasses", joinList, parseCommaSeparated),
    "Inline Github Issues": Metadata("Inline Github Issues", "inlineGithubIssues", joinValue, parseInlineGithubIssues),
    "Inline Tag Command": Metadata("Inline Tag Command", "inlineTagCommands", joinDict, parseInlineTagCommand),
    "Issue Class": Metadata("Issue Class", "issueClass", joinValue, parseLiteral),
    "Issue Tracker Template": Metadata("Issue Tracker Template", "issueTrackerTemplate", joinValue, parseLiteral),
    "Issue Tracking": Metadata("Issue Tracking", "issues", joinList, parseLinkedText),
    "Level": Metadata("Level", "level", joinValue, parseLevel),
    "Line Numbers": Metadata("Line Numbers", "lineNumbers", joinValue, parseBoolean),
    "Link Defaults": Metadata("Link Defaults", "linkDefaults", joinDdList, parseLinkDefaults),
    "Logo": Metadata("Logo", "logo", joinValue, parseLiteral),
    "Mailing List Archives": Metadata("Mailing List Archives", "mailingListArchives", joinValue, parseLiteral),
    "Mailing List": Metadata("Mailing List", "mailingList", joinValue, parseLiteral),
    "Markup Shorthands": Metadata("Markup Shorthands", "markupShorthands", joinBoolSet, parseMarkupShorthands),
    "Max Toc Depth": Metadata("Max ToC Depth", "maxToCDepth", joinValue, parseMaxToCDepth),
    "Metadata Include": Metadata("Metadata Include", "metadataInclude", joinBoolSet, partial(parseBoolishList, default=True)),
    "Metadata Order": Metadata("Metadata Order", "metadataOrder", joinValue, parseMetadataOrder),
    "No Abstract": Metadata("No Abstract", "noAbstract", joinValue, parseBoolean),
    "No Editor": Metadata("No Editor", "noEditor", joinValue, parseBoolean),
    "Note Class": Metadata("Note Class", "noteClass", joinValue, parseLiteral),
    "Opaque Elements": Metadata("Opaque Elements", "opaqueElements", joinList, parseCommaSeparated),
    "Prepare For Tr": Metadata("Prepare For Tr", "prepTR", joinValue, parseBoolean),
    "Previous Version": Metadata("Previous Version", "previousVersions", joinList, parseLiteralList),
    "Remove Multiple Links": Metadata("Remove Multiple Links", "removeMultipleLinks", joinValue, parseBoolean),
    "Repository": Metadata("Repository", "repository", joinValue, parseRepository),
    "Required Ids": Metadata("Required Ids", "requiredIDs", joinList, parseCommaSeparated),
    "Revision": Metadata("Revision", "level", joinValue, parseLevel),
    "Shortname": Metadata("Shortname", "displayShortname", joinValue, parseLiteral),
    "Slim Build Artifact": Metadata("Slim Build Artifact", "slimBuildArtifact", joinValue, parseBoolean),
    "Status Text": Metadata("Status Text", "statusText", joinList, parseLiteralList),
    "Status": Metadata("Status", "rawStatus", joinValue, parseLiteral),
    "Test Suite": Metadata("Test Suite", "testSuite", joinValue, parseLiteral),
    "Text Macro": Metadata("Text Macro", "customTextMacros", joinList, parseTextMacro),
    "Title": Metadata("Title", "title", joinValue, parseLiteral),
    "Toggle Diffs": Metadata("Toggle Diffs", "toggleDiffs", joinValue, parseBoolean),
    "TR": Metadata("TR", "TR", joinValue, parseLiteral),
    "Tracking Vector Class": Metadata("Tracking Vector Class", "trackingVectorClass", joinValue, parseLiteralOrNone),
    "Tracking Vector Image": Metadata("Tracking Vector Image", "trackingVectorImage", joinValue, parseLiteralOrNone),
    "Tracking Vector Image Width": Metadata("Tracking Vector Image Width", "trackingVectorImageWidth", joinValue, parseLiteral),
    "Tracking Vector Image Height": Metadata("Tracking Vector Image Height", "trackingVectorImageHeight", joinValue, parseLiteral),
    "Tracking Vector Alt Text": Metadata("Tracking Vector Alt Text", "trackingVectorAltText", joinValue, parseLiteral),
    "Tracking Vector Title": Metadata("Tracking Vector Title", "trackingVectorTitle", joinValue, parseLiteral),
    "Translate Ids": Metadata("Translate Ids", "translateIDs", joinDdList, parseTranslateIDs),
    "Translation": Metadata("Translation", "translations", joinList, parseTranslation),
    "URL": Metadata("URL", "ED", joinValue, parseLiteral),  # URL is a synonym for ED
    "Use <I> Autolinks": Metadata("Use <I> Autolinks", "useIAutolinks", joinValue, parseBoolean),
    "Use Dfn Panels": Metadata("Use Dfn Panels", "useDfnPanels", joinValue, parseBoolean),
    "Version History": Metadata("Version History", "versionHistory", joinList, parseLiteralList),
    "Warning": Metadata("Warning", "warning", joinValue, parseWarning),
    "Work Status": Metadata("Work Status", "workStatus", joinValue, parseWorkStatus),
    "Wpt Display": Metadata("Wpt Display", "wptDisplay", joinValue, parseWptDisplay),
    "Wpt Path Prefix": Metadata("Wpt Path Prefix", "wptPathPrefix", joinValue, parseLiteral)
}
