# coding: utf8
from __future__ import unicode_literals, print_function, division
import logging
from datetime import datetime
import re
import pkg_resources
from collections import Counter, defaultdict

import attr
import six
import git
from clldutils.dsv import reader
from clldutils.text import split_text_with_context
from clldutils.misc import lazyproperty
from clldutils.loglib import Logging
from clldutils.path import remove, rmtree, write_text
from clldutils import licenses
from clldutils import jsonlib
from pyglottolog.languoids import Glottocode

from segments.tokenizer import Tokenizer

import pylexibank
from pylexibank.util import DataDir, jsondump, textdump, get_badge
from pylexibank import cldf
from pylexibank import transcription

NOOP = -1


def non_empty(_, attribute, value):
    if not value:
        raise ValueError('{0} must be non-empty'.format(attribute))


class FieldnamesMixin(object):
    @classmethod
    def fieldnames(cls):
        return [f.name for f in attr.fields(cls)]


@attr.s(hash=False)
class Language(FieldnamesMixin):
    ID = attr.ib(default='', hash=True)
    Name = attr.ib(default=None)
    ISO639P3code = attr.ib(default=None)
    Glottocode = attr.ib(default=None)
    Macroarea = attr.ib(default=None)

    Glottolog_Name = attr.ib(default=None)
    Family = attr.ib(default=None)

    @classmethod
    def __cldf_table__(cls):
        return 'LanguageTable'


@attr.s(hash=False)
class Concept(FieldnamesMixin):
    ID = attr.ib(default='', hash=True)
    Name = attr.ib(default='')
    Concepticon_ID = attr.ib(default=None)
    Concepticon_Gloss = attr.ib(default=None)

    @classmethod
    def __cldf_table__(cls):
        return 'ParameterTable'


@attr.s
class Lexeme(FieldnamesMixin):
    """
    Raw lexical data item as it can be pulled out of the original datasets.

    This is the basis for creating rows in CLDF representations of the data by
    - splitting the lexical item into forms
    - cleaning the forms
    - potentially tokenizing the form
    """
    ID = attr.ib()
    Form = attr.ib()
    Value = attr.ib(validator=non_empty)  # the lexical item
    Language_ID = attr.ib(validator=non_empty)
    Parameter_ID = attr.ib(validator=non_empty)
    Local_ID = attr.ib(default=None)  # local ID of a lexeme in the source dataset
    Segments = attr.ib(
        default=attr.Factory(list),
        validator=attr.validators.instance_of(list))
    Source = attr.ib(
        default=attr.Factory(list),
        validator=attr.validators.instance_of(list),
        convert=lambda v: [v] if isinstance(v, six.string_types) else v)
    Comment = attr.ib(default=None)
    Cognacy = attr.ib(default=None)
    Loan = attr.ib(
        default=None,
        validator=attr.validators.optional(attr.validators.instance_of(bool)))

    @classmethod
    def __cldf_table__(cls):
        return 'FormTable'


@attr.s
class Cognate(FieldnamesMixin):
    ID = attr.ib(default=None)
    Form_ID = attr.ib(default=None)
    Form = attr.ib(default=None)
    Cognateset_ID = attr.ib(default=None)
    Doubt = attr.ib(
        default=False, convert=lambda v: v if isinstance(v, bool) else eval(v))
    Cognate_Detection_Method = attr.ib(default='expert')
    Source = attr.ib(
        default=attr.Factory(list),
        validator=attr.validators.instance_of(list),
        convert=lambda v: [v] if isinstance(v, six.string_types) else v)
    Alignment = attr.ib(
        default=None, convert=lambda v: ' '.join(v) if isinstance(v, list) else v)
    Alignment_Method = attr.ib(default=None)
    Alignment_Source = attr.ib(default=None)

    @classmethod
    def __cldf_table__(cls):
        return 'CognateTable'


@attr.s
class Metadata(object):
    title = attr.ib(default=None)
    description = attr.ib(default=None)
    citation = attr.ib(default=None)
    license = attr.ib(default=None)
    url = attr.ib(default=None)
    aboutUrl = attr.ib(default=None)
    conceptlist = attr.ib(default=None)
    lingpy_schema = attr.ib(default=None)
    derived_from = attr.ib(default=None)
    related = attr.ib(default=None)
    source = attr.ib(default=None)

    @property
    def common_props(self):
        return {
            "dc:title": self.title,
            "dc:description": self.description,
            "dc:bibliographicCitation": self.citation,
            "dc:license": licenses.find(self.license) if self.license else None,
            "dc:identifier": self.url,
            "dc:format": "http://concepticon.clld.org/contributions/{0}".format(
                self.conceptlist) if self.conceptlist else None,
            "dc:isVersionOf": "http://lexibank.clld.org/contributions/{0}".format(
                self.derived_from) if self.derived_from else None,
            "dc:related": self.related,
            "aboutUrl": self.aboutUrl,
        }


class Dataset(object):
    """
    A lexibank dataset.

    This object provides access to a dataset's
    - language list as attribute `languages`
    - concept list as attribute `concepts`
    - concepticon concept-list ID as attribute `conceptlist`
    """
    dir = None  # Derived classes must provide an existing directory here!
    lexeme_class = Lexeme
    cognate_class = Cognate
    language_class = Language
    concept_class = Concept
    log = logging.getLogger(pylexibank.__name__)

    @lazyproperty
    def id(self):
        return self.dir.name

    @lazyproperty
    def metadata(self):
        return Metadata(**jsonlib.load(self.dir / 'metadata.json'))

    @property
    def stats(self):
        if self.dir.joinpath('README.json').exists():
            return jsonlib.load(self.dir / 'README.json')
        return {}

    def __init__(self, concepticon=None, glottolog=None):
        self.unmapped = Unmapped()
        self.dir = DataDir(self.dir)
        self._json = self.dir.joinpath('lexibank.json')
        self.raw = DataDir(self.dir / 'raw')
        self.raw.mkdir(exist_ok=True)
        self.cldf_dir = self.dir / 'cldf'
        self.cldf_dir.mkdir(exist_ok=True)

        self.conceptlist = {}
        self.glottolog = glottolog
        self.concepticon = concepticon
        try:
            self.git_repo = git.Repo(str(self.dir))  # pragma: no cover
        except git.InvalidGitRepositoryError:
            self.git_repo = None
        self.tr_analyses = {}
        self.tr_bad_words = []
        self.tr_invalid_words = []

    def _iter_etc(self, what):
        path = self.dir / 'etc' / what
        return reader(path, dicts=True) if path.exists() else []

    def read_json(self):
        return jsonlib.load(self._json) if self._json.exists() else {}

    def write_json(self, obj):
        jsondump(obj, self._json)

    @lazyproperty
    def github_repo(self):  # pragma: no cover
        try:
            match = re.search(
                'github\.com/(?P<org>[^/]+)/%s\.git' % re.escape(self.id),
                self.git_repo.remotes.origin.url)
            if match:
                return match.group('org') + '/' + self.id
        except AttributeError:
            pass

    @lazyproperty
    def sources(self):
        return list(self._iter_etc('sources.csv'))

    @lazyproperty
    def concepts(self):
        return list(self._iter_etc('concepts.csv'))

    @lazyproperty
    def languages(self):
        res = []
        for item in self._iter_etc('languages.csv'):
            if item.get('GLOTTOCODE', None) and not \
                    Glottocode.pattern.match(item['GLOTTOCODE']):  # pragma: no cover
                raise ValueError(
                    "Wrong glottocode for item {0}".format(item['GLOTTOCODE']))
            res.append(item)
        return res

    @lazyproperty
    def lexemes(self):
        res = {}
        for item in self._iter_etc('lexemes.csv'):
            res[item['LEXEME']] = item['REPLACEMENT']
        return res

    # ---------------------------------------------------------------
    # workflow actions whih should be overwritten by derived classes:
    # ---------------------------------------------------------------
    def cmd_download(self, **kw):
        self._not_implemented('download')
        return NOOP

    def cmd_install(self, **kw):
        self._not_implemented('install')
        return NOOP

    # ---------------------------------------------------------------
    # handling of lexemes/forms/words
    # ---------------------------------------------------------------
    def clean_form(self, item, form):
        """
        Called when a row is added to a CLDF dataset.

        :param form:
        :return: None to skip the form, or the cleaned form as string.
        """
        if form not in ['?']:
            return form

    def split_forms(self, item, value):
        if value in self.lexemes:
            self.log.debug('overriding via lexemes.csv: %r -> %r' % (value, self.lexemes[value]))
        value = self.lexemes.get(value, value)
        return [self.clean_form(item, form)
                for form in split_text_with_context(value, separators='/,;')]

    @lazyproperty
    def tokenizer(self):
        """
        Datasets can provide support for segmentation (aka tokenization) in two ways:
        - by providing an orthography profile at etc/orthography.tsv or
        - by overwriting this method to return a custom tokenizer callable.

        :return: A callable to do segmentation.

        The expected signature of the callable is

            def t(item, string, **kw)

        where
        - `item` is a `dict` representing the complete CLDF FormTable row
        - `string` is the string to be segmented
        - `kw` may be used to pass any context info to the tokenizer, when called
          explicitly.
        """
        profile = self.dir / 'etc' / 'orthography.tsv'
        if profile.exists():
            obj = Tokenizer(profile=str(profile))

            def _tokenizer(item, string, **kw):
                kw.setdefault("column", "IPA")
                kw.setdefault("separator", " _ ")
                return obj(string, **kw).split()
            return _tokenizer

    # ---------------------------------------------------------------
    # CLDF dataset access
    # ---------------------------------------------------------------
    @lazyproperty
    def cldf(self):
        return cldf.Dataset(self)

    # ---------------------------------------------------------------
    def _download(self, **kw):
        self.cmd_download(**kw)
        write_text(
            self.raw / 'README.md',
            'Raw data downloaded {0}'.format(datetime.utcnow().isoformat()))

    def _install(self, **kw):
        self.log = kw.get('log', self.log)
        self.unmapped.clear()
        rmtree(self.cldf_dir)
        self.cldf_dir.mkdir()
        self.tr_analyses = {}
        self.tr_bad_words = []
        self.tr_invalid_words = []

        if self.metadata.conceptlist:
            self.conceptlist = self.concepticon.conceptlists[self.metadata.conceptlist]
        if self.cmd_install(**kw) == NOOP:
            return

        if kw.get('verbose'):
            self.unmapped.pprint()
        self.cldf.validate(kw['log'])

        stats = transcription.Stats(
            bad_words=sorted(self.tr_bad_words[:100], key=lambda x: x['ID']),
            bad_words_count=len(self.tr_bad_words),
            invalid_words=sorted(self.tr_invalid_words[:100], key=lambda x: x['ID']),
            invalid_words_count=len(self.tr_invalid_words))
        for lid, analysis in self.tr_analyses.items():
            for attribute in ['segments', 'bipa_errors', 'sclass_errors', 'replacements']:
                getattr(stats, attribute).update(getattr(analysis, attribute))
            stats.general_errors += analysis.general_errors
            stats.inventory_size += len(analysis.segments) / len(self.tr_analyses)

        error_segments = stats.bipa_errors.union(stats.sclass_errors)
        for i, row in enumerate(stats.bad_words):
            analyzed_segments = []
            for s in row['Segments']:
                analyzed_segments.append('<s> %s </s>' % s if s in error_segments else s)
            stats.bad_words[i] = [
                row['ID'],
                row['Language_ID'],
                row['Parameter_ID'],
                row['Form'],
                ' '.join(analyzed_segments)]

        for i, row in enumerate(stats.invalid_words):
            stats.invalid_words[i] = [
                row['ID'],
                row['Language_ID'],
                row['Parameter_ID'],
                row['Form']]
        # Aggregate transcription analysis results ...
        tr = dict(
            by_language={k: attr.asdict(v) for k, v in self.tr_analyses.items()},
            stats=attr.asdict(stats))
        # ... and write a report:
        for text, fname in [
            (transcription.report(tr), 'TRANSCRIPTION.md'),
            (self.report(tr, log=kw.get('log')), 'README.md'),
        ]:
            textdump(text, self.dir / fname, log=kw.get('log'))

    def _clean(self, **kw):
        self.log.debug('removing CLDF directory %s' % self.cldf_dir)
        if self.cldf_dir.exists():
            for f in self.cldf_dir.iterdir():
                if f.is_file():
                    remove(f)
                else:
                    rmtree(f)

    def _not_implemented(self, method):
        self.log.warn('cmd_{0} not implemented for dataset {1}'.format(method, self.id))

    def coverage(self, vars, glangs, c):  # pragma: no cover
        for row in self.cldf['FormTable']:
            try:
                cid = int(row['Parameter_ID'])
            except (ValueError, TypeError):
                continue
            vid = self.id + '-' + row['Language_ID']
            c[cid].add(vid)
            vars[vid].add(cid)
            glangs[row['Language_ID']].add(cid)

    def build_status_badge(self):
        if not self.dir.joinpath('.travis.yml').exists():
            return ''
        try:
            return "[![Build Status](https://travis-ci.org/{0}.svg?branch=master)]" \
                   "(https://travis-ci.org/{0})".format(self.github_repo)
        except:
            return ''

    def report(self, tr_analysis, log=None):
        #
        # FIXME: write only summary into README.md
        # in case of multiple cldf datasets:
        # - separate lexemes.md and transcriptions.md
        #
        if not list(self.cldf_dir.glob('*.csv')):
            return
        lines = [
            '# %s\n' % self.metadata.title,
            'Cite the source dataset as\n',
            '> %s\n' % self.metadata.citation,
        ]

        if self.metadata.license:
            lines.extend([
                'This dataset is licensed under a %s license' % self.metadata.license, ''])

        if self.metadata.url:
            lines.extend(['Available online at %s' % self.metadata.url, ''])

        if self.metadata.related:
            lines.extend(['See also %s' % self.metadata.related, ''])

        if self.metadata.conceptlist:
            lines.extend([
                'Conceptlist in Concepticon: [{0}](http://concepticon.clld.org/contributions'
                '/{0})'.format(self.metadata.conceptlist),
                ''])

        # add NOTES.md
        if self.dir.joinpath('NOTES.md').exists():
            lines.extend(['## Notes', ''])
            lines.extend(self.dir.joinpath('NOTES.md').read_text().split("\n"))
            lines.extend(['', ''])  # some blank lines

        synonyms = defaultdict(Counter)
        totals = {
            'languages': Counter(),
            'concepts': Counter(),
            'sources': Counter(),
            'cognate_sets': Counter(),
            'lexemes': 0,
        }

        missing_source = []
        missing_lang = []

        for row in self.cldf['FormTable']:
            if row['Source']:
                totals['sources'].update(['y'])
            else:
                missing_source.append(row)
            totals['concepts'].update([row['Parameter_ID']])
            totals['languages'].update([row['Language_ID']])
            totals['lexemes'] += 1
            synonyms[row['Language_ID']].update([row['Parameter_ID']])

        for row in self.cldf['CognateTable']:
            totals['cognate_sets'].update([row['Cognateset_ID']])

        sindex = sum(
            [sum(list(counts.values())) / float(len(counts)) for counts in synonyms.values()])
        langs = set(synonyms.keys())
        if langs:
            sindex /= float(len(langs))
        else:
            sindex = 0
        totals['SI'] = sindex

        stats = tr_analysis['stats']
        lsegments = len(stats['segments'])
        lbipapyerr = len(stats['bipa_errors'])
        lsclasserr = len(stats['sclass_errors'])

        def ratio(prop):
            return sum(v for k, v in totals[prop].items() if k) / float(totals['lexemes'])

        badges = [
            self.build_status_badge(),
            get_badge(ratio('languages'), 'Glottolog'),
            get_badge(ratio('concepts'), 'Concepticon'),
            get_badge(ratio('sources'), 'Source'),
        ]
        if lsegments:
            badges.extend([
                get_badge((lsegments - lbipapyerr) / lsegments, 'BIPA'),
                get_badge((lsegments - lsclasserr) / lsegments, 'CLTS SoundClass'),
            ])
        lines.extend(['## Statistics', '\n', '\n'.join(badges), ''])
        stats_lines = [
            '- **Varieties:** {0:,}'.format(len(totals['languages'])),
            '- **Concepts:** {0:,}'.format(len(totals['concepts'])),
            '- **Lexemes:** {0:,}'.format(totals['lexemes']),
            '- **Synonymy:** %.2f' % (totals['SI']),
            '- **Cognacy:** {0:,} cognates in {1:,} cognate sets'.format(
                sum(v for k, v in totals['cognate_sets'].items() if v > 1),
                sum(1 for k, v in totals['cognate_sets'].items() if v > 1)),
            '- **Invalid lexemes:** {0:,}'.format(stats['invalid_words_count']),
            '- **Tokens:** {0:,}'.format(sum(stats['segments'].values())),
            '- **Segments:** {0:,} ({1} BIPA errors, {2} CTLS sound class errors, {3} CLTS modified)'
                .format(lsegments, lbipapyerr, lsclasserr, len(stats['replacements'])),
            '- **Inventory size (avg):** %.2f' % stats['inventory_size'],
            ]
        if log:
            log.info('\n'.join(['Summary for dataset {}'.format(self.id)] + stats_lines))
        lines.extend(stats_lines)

        totals['languages'] = len(totals['languages'])
        totals['concepts'] = len(totals['concepts'])
        totals['cognate_sets'] = bool(1 for k, v in totals['cognate_sets'].items() if v > 1)
        totals['sources'] = totals['sources'].get('y', 0)

        bookkeeping_languoids = []
        for lang in self.cldf['LanguageTable']:
            gl_lang = self.glottolog.cached_languoids.get(lang.get('Glottocode'))
            if gl_lang and gl_lang.category == 'Bookkeeping':
                bookkeeping_languoids.append(lang)

        # improvements section
        if missing_lang or missing_source or bookkeeping_languoids:
            lines.extend(['\n## Possible Improvements:\n', ])

            if missing_lang:
                lines.append("- Languages missing glottocodes: %d/%d (%.2f%%)" % (
                    len(missing_lang),
                    totals['languages'],
                    (len(missing_lang) / totals['languages']) * 100
                ))

            if bookkeeping_languoids:
                lines.append(
                    "- Languages linked to [bookkeeping languoids in Glottolog]"
                    "(http://glottolog.org/glottolog/glottologinformation"
                    "#bookkeepinglanguoids):")
            for lang in bookkeeping_languoids:
                lines.append(
                    '  - {0} [{1}](http://glottolog.org/resource/languoid/id/{1})'.format(
                        lang.get('Name', lang.get('ID')), lang['Glottocode']))
            lines.append('\n')

        if missing_source:
            lines.append("- Entries missing sources: %d/%d (%.2f%%)" % (
                len(missing_source),
                totals['lexemes'],
                (len(missing_source) / totals['lexemes']) * 100
            ))

        return lines


MARKDOWN_TEMPLATE = """
## Transcription Report

### General Statistics

* Number of Tokens: {tokens}
* Number of Segments: {segments}
* Invalid forms: {invalid}
* Inventory Size: {inventory_size:.2f}
* [Erroneous tokens](report.md#tokens): {general_errors}
* Erroneous words: {word_errors}
* Number of BIPA-Errors: {bipa_errors}
* Number of CLTS-SoundClass-Errors: {sclass_errors}
* Bad words: {words_errors}
"""


class Unmapped(object):
    def __init__(self):
        self.languages = set()
        self.concepts = set()

    def clear(self):
        self.languages = set()
        self.concepts = set()

    def add_concept(self, **kw):
        self.concepts.add(Concept(**kw))

    def add_language(self, **kw):
        self.languages.add(Language(**kw))

    @staticmethod
    def quote(v):
        v = '{0}'.format(v or '')
        if ',' in v or len(v.split()) > 1:
            v = '"%s"' % v.replace('"', '""')
        return v

    def pprint(self):
        for objs, cls in [(self.languages, Language), (self.concepts, Concept)]:
            if objs:
                print('=== Unmapped %ss ===' % cls.__name__)
                print(','.join([a.name.upper() for a in attr.fields(cls)]))
                for row in sorted(map(attr.astuple, objs)):
                    print(','.join(map(self.quote, row)))


def iter_datasets(glottolog=None, concepticon=None, verbose=False):
    for ep in pkg_resources.iter_entry_points('lexibank.dataset'):
        try:
            yield ep.load()(glottolog=glottolog, concepticon=concepticon)
        except ImportError as e:
            if verbose:
                print('Importing {0} failed: {1}'.format(ep.name, e))
