import gzip
import re
import typing as T
from collections import defaultdict
from contextlib import ExitStack, contextmanager
from enum import Enum
from logging import getLogger
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Dict, Iterable, List, NamedTuple, Optional

from sqlalchemy import MetaData, create_engine, not_
from sqlalchemy.engine import Engine
from sqlalchemy.orm import create_session, exc, sessionmaker

from cumulusci.salesforce_api.filterable_objects import NOT_COUNTABLE, NOT_EXTRACTABLE
from cumulusci.salesforce_api.org_schema_models import (
    Base,
    Field,
    FileMetadata,
    SObject,
)
from cumulusci.utils.fileutils import FSResource
from cumulusci.utils.http.multi_request import (
    RECOVERABLE_ERRORS,
    CompositeParallelSalesforce,
)
from cumulusci.utils.salesforce.count_sobjects import count_sobjects

y2k = "Sat, 1 Jan 2000 00:00:01 GMT"


def zip_database(tempfile: Path, schema_path: T.Union[FSResource, Path]):
    """Compress tempfile.db to schema_path.db.gz"""
    with tempfile.open("rb") as db:
        with schema_path.open("wb") as fileobj:
            with gzip.GzipFile(fileobj=fileobj, mode="w") as gzipped:
                gzipped.write(db.read())


def unzip_database(gzipfile, outfile):
    """Decompress schema_path.db.gz to outfile.db"""
    with gzipfile.open("rb") as fileobj:
        with gzip.GzipFile(fileobj=fileobj) as gzipped:
            with open(outfile, "wb") as db:
                db.write(gzipped.read())


def ignore_based_on_name(objname, patterns: T.Sequence[re.Pattern]):
    return any(pat.fullmatch(objname) for pat in patterns)


# TODO: Rename this SObjectFilters
class Filters(Enum):
    """Options for filtering schemas"""

    # These were originally done using SQL Alchemy syntax because filtering
    # was done in SQL Alchemy. It is still possible to use them that way
    # in client code:
    #
    # schema.objects.filter(Filters.activateable)
    activateable = SObject.activateable
    compactLayoutable = SObject.compactLayoutable
    createable = SObject.createable
    deepCloneable = SObject.deepCloneable
    deletable = SObject.deletable
    layoutable = SObject.layoutable
    listviewable = SObject.listviewable
    lookupLayoutable = SObject.lookupLayoutable
    mergeable = SObject.mergeable
    queryable = SObject.queryable
    replicateable = SObject.replicateable
    retrieveable = SObject.retrieveable
    searchLayoutable = SObject.searchLayoutable
    searchable = SObject.searchable
    triggerable = SObject.triggerable
    undeletable = SObject.undeletable
    updateable = SObject.updateable

    not_activateable = not_(SObject.activateable)
    not_compactLayoutable = not_(SObject.compactLayoutable)
    not_createable = not_(SObject.createable)
    not_deepCloneable = not_(SObject.deepCloneable)
    not_deletable = not_(SObject.deletable)
    not_layoutable = not_(SObject.layoutable)
    not_listviewable = not_(SObject.listviewable)
    not_lookupLayoutable = not_(SObject.lookupLayoutable)
    not_mergeable = not_(SObject.mergeable)
    not_queryable = not_(SObject.queryable)
    not_replicateable = not_(SObject.replicateable)
    not_retrieveable = not_(SObject.retrieveable)
    not_searchLayoutable = not_(SObject.searchLayoutable)
    not_searchable = not_(SObject.searchable)
    not_triggerable = not_(SObject.triggerable)
    not_undeletable = not_(SObject.undeletable)
    not_updateable = not_(SObject.updateable)

    extractable = "extractable"  # can it be extracted safely?
    # non-extractable objects are discovered
    # through experimentation.
    # Also, all non-queryable and non-retrievable objects are
    # considered non-extractable.

    populated = SObject.count > 0  # does it have data in the org?


# TODO: Profiling and optimizing of the
#      SQL parts. After the object is frozen,
#      all query-sets can be cached as
#      dicts and lists
class Schema:
    """Represents an org's schema, cached from describe() calls"""

    included_objects = None
    includes_counts = False

    def __init__(self, engine, schema_path, filters: T.Sequence[Filters] = ()):
        self.engine = engine
        Session = sessionmaker(bind=self.engine)
        self.session = Session()
        self.path = schema_path
        self.filters = set(filters)

    @property
    def sobjects(self):
        query = self.session.query(SObject)
        if self.included_objects is not None:
            query = query.filter(SObject.name.in_(self.included_objects))
        return query

    def __getitem__(self, name):
        try:
            return self.sobjects.filter_by(name=name).one()
        except exc.NoResultFound:
            raise KeyError(f"No sobject named `{name}`")

    def __contains__(self, name):
        return bool(self.sobjects.filter_by(name=name).first())

    def keys(self):
        return [x.name for x in self.sobjects.all()]

    def values(self):
        return self.sobjects.all()

    def items(self):
        return [(obj.name, obj) for obj in self.sobjects]

    def get(self, name: str):
        return self.sobjects.filter_by(name=name).first()

    def block_writing(self):
        """After this method is called, the database can't be updated again"""
        # changes don't get saved back to the gzip
        # so there is no point writing to the DB
        def closed():
            raise IOError("Database is not open for writing")

        self.session._real_commit__ = self.session.commit
        self.session.commit = closed

    def close(self):
        self.session.close()

    def __repr__(self):
        return f"<Schema {self.path} : {self.engine}>"

    def add_counts(self, counts: T.Dict[str, int]):
        for objname, count in counts.items():
            obj = self.get(objname)
            if obj:
                obj.count = count
        self.includes_counts = True

    def populate_cache(
        self,
        sf,
        included_objects,
        filters: T.Sequence[Filters] = (),
        patterns_to_ignore: T.Sequence[str] = (),
        logger=None,
        *,
        include_counts: bool = False,
    ) -> T.Union[T.Dict[str, int], T.Dict[str, None]]:
        """Populate a schema cache from the API, using last_modified_date
        to pull down only new schema"""
        for pat in patterns_to_ignore:
            assert pat.replace("%", "").isidentifier(), f"Pattern has wrong chars {pat}"

        # Platform bug!
        patterns_to_ignore = ("MacroInstruction%",) + tuple(patterns_to_ignore)
        regexps_to_ignore = [
            re.compile(pat.replace("%", ".*"), re.IGNORECASE)
            for pat in patterns_to_ignore
        ]

        objs = [obj for obj in sf.describe()["sobjects"]]
        if included_objects:
            objs = [obj for obj in objs if obj["name"] in included_objects]
        sobj_names = [
            obj["name"]
            for obj in objs
            if not (
                ignore_based_on_name(obj["name"], regexps_to_ignore)
                or ignore_based_on_properties(obj, filters)
            )
        ]

        def find_last_modified_date(sobj_name: str) -> Optional[str]:
            if obj := self.get(sobj_name):
                return obj.last_modified_date
            else:
                return y2k

        objs_to_refresh = {
            sobj_name: find_last_modified_date(sobj_name) for sobj_name in sobj_names
        }
        changes = list(deep_describe(sf, objs_to_refresh, logger))

        self._populate_cache_from_describe(changes)
        if include_counts:
            results = populate_counts(sf, self, sobj_names, logger)
        else:
            results = {name: None for name in sobj_names}
        return results

    def _populate_cache_from_describe(self, describe_objs: List["DescribeUpdate"]):
        """Populate a schema cache from a list of describe objects."""
        engine = self.engine
        metadata = Base.metadata
        metadata.bind = engine
        metadata.reflect()

        with BufferedSession(engine, metadata) as sess:

            for (sobj_data, last_modified) in describe_objs:
                sobj_data = sobj_data.copy()
                fields = sobj_data.pop("fields")
                sobj_data["last_modified_date"] = last_modified
                create_row(sess, SObject, sobj_data)
                for field in fields:
                    field["sobject"] = sobj_data["name"]
                    create_row(sess, Field, field)

            self.save_version(sess)

        engine.execute("vacuum")

    FormatVersion = "FormatVersion"
    CurrentFormatVersion = 2

    @property
    def version(self) -> int:
        rows = self.session.query(FileMetadata.value)
        rows = rows.filter(FileMetadata.name == self.FormatVersion)
        first_row = rows.one()
        assert first_row
        return int(first_row[0])

    def save_version(self, sess: "BufferedSession"):
        create_row(
            sess,
            FileMetadata,
            {"name": self.FormatVersion, "value": self.CurrentFormatVersion},
        )
        self.session.commit()


def create_row(buffered_session: "BufferedSession", model, valuesdict: dict):
    buffered_session.write_single_row(model.__tablename__, valuesdict)


class SilentMigration(Exception):
    pass


class BufferedSession:
    """Buffer writes to a SQL DB for faster performmance"""

    def __init__(self, engine: Engine, metadata: MetaData, max_buffer_size: int = 1000):
        self.buffered_rows = defaultdict(list)
        self.columns = {}
        self.engine = engine
        self.metadata = metadata
        self._prepare()
        self.max_buffer_size = max_buffer_size

    def __enter__(self, *args):
        self.session = create_session(bind=self.engine, autocommit=False)
        return self

    def __exit__(self, *args):
        self.close()

    def _prepare(self):
        # Setup table info used by the write-buffering infrastructure
        assert self.metadata.tables
        self.insert_statements = {}
        for tablename, model in self.metadata.tables.items():
            self.insert_statements[tablename] = model.insert(bind=self.engine)
            self.columns[tablename] = {
                colname: None for colname in model.columns.keys()
            }

    def write_single_row(self, tablename: str, row: Dict) -> None:
        # but first, normalize it so all keys have a value. SQLite Requires it.
        normalized_row = {}
        for key, default in self.columns[tablename].items():
            normalized_row[key] = row.get(key, default)

        # cache the value for later insert
        self.buffered_rows[tablename].append(normalized_row)

        # flush if buffer is full
        if len(self.buffered_rows[tablename]) > self.max_buffer_size:
            self.flush()

    def flush(self):
        for tablename, insert_statement in self.insert_statements.items():
            # seems much more efficient than line-by-line inserts.
            if self.buffered_rows[tablename]:
                self.session.execute(insert_statement, self.buffered_rows[tablename])
                self.buffered_rows[tablename] = []
        self.session.flush()

    def commit(self):
        self.flush()
        self.session.commit()

    def close(self) -> None:
        self.commit()
        self.session.close()


@contextmanager
def get_org_schema(
    sf,
    org_config,
    *,
    include_counts: bool = False,
    filters: T.Sequence[Filters] = (),
    patterns_to_ignore: T.Tuple[str, ...] = (),
    included_objects: T.List[str] = (),
    force_recache=False,
    logger=None,
):
    """
    Get a read-only representation of an org's schema.
    sf - simple_saleforce object
    org_config - an OrgConfig for the relevant org

    include_counts: query each queryable/retrievable object count.
                    This takes time and may even timeout in huge orgs!
    filters: A sequence of Filters which are the same as Salesforce SObject properties
            like .createable, .deletable etc. Objects that do not match are ignored.
            Two special filters exist:
                * Filters.extractable, which uses heuristics to limit to objects
                    that extract properly
                * Filters.populated, which limits to objects that have data in them.
                    This depends upon `include_counts`
    included_objects: Ignore objects not in this list. Stacks with other filters
    patterns_to_ignore: Strings in SQL %LIKE% syntax that match SObjects to be
                        ignored.

    force_recache: True - replace cache. False (default) - use/update cache is available.
    logger - replace the standard logger "cumulusci.salesforce_api.org_schema"

    This function take 5-20 seconds (or even more) depending on
    the complexity of the org. Call it at most once per task!
    """
    assert not isinstance(patterns_to_ignore, str)

    filters = set(filters)
    with org_config.get_orginfo_cache_dir(Schema.__module__) as directory:
        directory.mkdir(exist_ok=True, parents=True)
        schema_path = directory / "org_schema.db.gz"

        if force_recache and schema_path.exists():
            schema_path.unlink()

        if Filters.populated in filters:
            filters.add(Filters.queryable)
            filters.add(Filters.retrieveable)
            # experiment with removing this limitation by using limit 1 query instead
            assert include_counts, "Filters.populated depends on include_counts"
            patterns_to_ignore += NOT_COUNTABLE

        if Filters.extractable in filters:
            filters.add(Filters.queryable)
            filters.add(Filters.retrieveable)
            filters.add(Filters.createable)  # so we can load again later
            patterns_to_ignore += NOT_EXTRACTABLE

        logger = logger or getLogger(__name__)

        with ZippableTempDb() as tempdb, ExitStack() as closer:
            schema = None
            engine = tempdb.create_engine()
            if schema_path.exists():
                try:
                    cleanups_on_failure = []
                    tempdb.unzip_database(schema_path)
                    cleanups_on_failure.extend([schema_path.unlink, tempdb.clear])
                    schema = Schema(engine, schema_path, filters)

                    cleanups_on_failure.append(schema.close)
                    closer.callback(schema.close)
                    if schema.version != schema.CurrentFormatVersion:
                        raise SilentMigration(
                            "was created with older CumulusCI version"
                        )
                    assert schema.sobjects.first().name
                    schema.from_cache = True
                except Exception as e:
                    if not isinstance(e, SilentMigration):
                        logger.warning(
                            f"Cannot read `{schema_path}`. Recreating it. Reason `{e}`."
                        )
                    schema = None
                    for cleanup_action in reversed(cleanups_on_failure):
                        cleanup_action()

            if schema is None:
                Base.metadata.bind = engine
                Base.metadata.create_all()
                schema = Schema(engine, schema_path, filters)
                closer.callback(schema.close)
                schema.from_cache = False

            populated_objs = schema.populate_cache(
                sf,
                included_objects,
                filters,
                patterns_to_ignore,
                logger,
                include_counts=include_counts,
            )

            if Filters.populated in filters:
                # another way to compute this might be by querying the first ID
                objs_to_include = [
                    objname for objname, count in populated_objs.items() if count > 0
                ]
            else:
                objs_to_include = [objname for objname, _ in populated_objs.items()]

            schema.included_objects = objs_to_include
            schema.block_writing()
            # save a gzipped copy for later
            tempdb.zip_database(schema_path)
            yield schema


class ZippableTempDb:
    """A database that loads and saves from a tempdir to a zippped cache"""

    def __enter__(self) -> "ZippableTempDb":
        self.tempdir = TemporaryDirectory()
        self.tempfile = Path(self.tempdir.name) / "temp_org_schema.db"
        return self

    def __exit__(self, *args, **kwargs):
        self.clear()
        self.tempdir.cleanup()

    def zip_database(self, target_path: T.Union[FSResource, Path]):
        "Save a gzipped copy for later"
        zip_database(self.tempfile, target_path)

    def unzip_database(self, zipped_db: T.Union[FSResource, Path]):
        unzip_database(zipped_db, self.tempfile)

    def clear(self):
        if self.tempfile.exists():
            self.tempfile.unlink()

    def create_engine(self):
        return create_engine(f"sqlite:///{str(self.tempfile)}")


def populate_counts(sf, schema, objs_cached, logger) -> T.Dict[str, int]:
    objects_to_count = [objname for objname in objs_cached]
    counts, transports_errors, salesforce_errors = count_sobjects(sf, objects_to_count)
    errors = transports_errors + salesforce_errors
    for error in errors[0:10]:
        logger.warning(f"Error counting SObjects: {error}")

    if len(errors) > 10:
        logger.warning(f"{len(errors)} more counting errors suppressed")

    schema.add_counts(counts)
    schema.session.flush()
    return counts


class DescribeResponse(NamedTuple):
    """Result of a describe call from Salesforce"""

    status: int
    body: dict
    last_modified_date: str = None


class DescribeUpdate(NamedTuple):
    body: dict
    last_modified_date: str = None


lm_date = Optional[str]


def deep_describe(sf, objs: Dict[str, lm_date], logger) -> Iterable[DescribeUpdate]:
    """Fetch describe data for changed sobjects

    Fetch describe data for sobjects from the list 'objs'
    which have changed since last_modified_date (in HTTP
    proto format) and yield each object as a DescribeResponse object."""

    logger = logger or getLogger(__name__)
    with CompositeParallelSalesforce(sf, max_workers=8) as cpsf:
        responses, errors = cpsf.do_composite_requests(
            (
                {
                    "method": "GET",
                    "url": f"/services/data/v{sf.sf_version}/sobjects/{obj}/describe",
                    "referenceId": f"ref{obj}",
                    "httpHeaders": {"If-Modified-Since": last_modified_date or y2k},
                }
                for obj, last_modified_date in objs.items()
            )
        )

        for error in errors[0:5]:
            logger.warning(f"Error calling Salesforce API: {error}")
        if len(errors) > 5:
            logger.warning(f"...Total {len(errors)} errors")

        unrecoverable_errors = (
            error
            for error, matching_request in errors
            if not isinstance(error, RECOVERABLE_ERRORS)
        )
        first_unrecoverable_error = next(unrecoverable_errors, None)
        if first_unrecoverable_error:
            raise first_unrecoverable_error

        responses = (
            DescribeResponse(
                response["httpStatusCode"],
                response["body"],
                response["httpHeaders"].get("Last-Modified"),
            )
            for response in responses
        )

        changes = [
            DescribeUpdate(resp.body, resp.last_modified_date)
            for resp in responses
            if resp.status == 200
        ]
        unexpected = [resp for resp in responses if resp.status not in (200, 304)]
        for unknown in unexpected:  # pragma: no cover
            logger.warning(
                f"Unexpected describe reply. An SObject may be missing: {unknown}"
            )

        yield from changes


def ignore_based_on_properties(obj: dict, filters: T.Sequence[Filters]):
    return not all(obj.get(filter.name, True) for filter in filters)
