################################################################################
## Dactyl Page Class
##
## Handles the loading, default values, preprocessing, and content parsing
## for a single HTML page of output.
################################################################################

import jinja2
import requests

from markdown import markdown
from bs4 import BeautifulSoup

from dactyl.common import *

from dactyl.jinja_loaders import FrontMatterRemoteLoader, FrontMatterFSLoader

class DactylPage:
    def __init__(self, config, data, skip_pp=False, more_filters=[]):
        self.config = config
        self.data = data
        self.rawtext = None
        self.pp_template = None
        self.twolines = None
        self.toc = []
        self.ffp = None
        self.skip_pp = skip_pp
        self.md = None
        self.html = None
        self.soup = None

        self.gain_filters(more_filters)
        self.load_content()
        self.provide_default_filename()
        self.provide_name()
        # self.html_content({"currentpage":data, **context})

    def get_pp_env(self, loader):
        if (self.config["preprocessor_allow_undefined"] or
                self.config.bypass_errors):
            preferred_undefined = jinja2.Undefined
        else:
            preferred_undefined = jinja2.StrictUndefined

        pp_env = jinja2.Environment(undefined=preferred_undefined,
                extensions=['jinja2.ext.i18n'], loader=loader)

        # Add custom "defined_and_" tests
        def defined_and_equalto(a,b):
            return pp_env.tests["defined"](a) and pp_env.tests["equalto"](a, b)
        pp_env.tests["defined_and_equalto"] = defined_and_equalto
        def undefined_or_ne(a,b):
            return pp_env.tests["undefined"](a) or pp_env.tests["ne"](a, b)
        pp_env.tests["undefined_or_ne"] = undefined_or_ne

        # Pull exported values (& functions) from page filters into the pp_env
        for filter_name in self.filters(save=False):
            if filter_name not in self.config.filters.keys():
                logger.debug("Skipping unloaded filter '%s'" % filter_name)
                continue
            if "export" in dir(self.config.filters[filter_name]):
                for key,val in self.config.filters[filter_name].export.items():
                    logger.debug("... pulling in filter_%s's exported key '%s'"
                            % (filter_name, key))
                    pp_env.globals[key] = val

        return pp_env

    def load_content(self):
        """
        Dispatcher for loading this page's content based on the "md" field.
        """
        logger.debug("Loading page %s" % self)
        if "md" in self.data:
            if (self.data["md"][:5] == "http:" or
                    self.data["md"][:6] == "https:"):
                self.load_from_url()
            else:
                self.load_from_disk()
        elif "__md_generator" in self.data:
            self.load_from_generator()

    def load_from_url(self):
        """
        Read file over HTTP(S),
        as either raw text or as a Jinja template,
        and load frontmatter, if any, either way.
        """
        url = self.data["md"]
        logger.info("Loading page from URL: %s"%url)
        assert (url[:5] == "http:" or url[:6] == "https:")
        if not self.skip_pp:
            pp_env = self.get_pp_env(loader=FrontMatterRemoteLoader())
            self.pp_template = pp_env.get_template(self.data["md"])
            frontmatter = pp_env.loader.fm_map[self.data["md"]]
            merge_dicts(frontmatter, self.data)
            # special case: let frontmatter overwrite default "html" vals
            if PROVIDED_FILENAME_KEY in self.data and "html" in frontmatter:
                self.data["html"] = frontmatter["html"]
            self.twolines = pp_env.loader.twolines[self.data["md"]]
        else:
            response = requests.get(url)
            if response.status_code == 200:
                self.rawtext, frontmatter = parse_frontmatter(response.text)
                merge_dicts(frontmatter, self.data)
                # special case: let frontmatter overwrite default "html" vals
                if PROVIDED_FILENAME_KEY in self.data and "html" in frontmatter:
                    self.data["html"] = frontmatter["html"]
                self.twolines = self.rawtext.split("\n", 2)[:2]
            else:
                raise requests.RequestException("Status code for page was not 200")

    def load_from_disk(self):
        """
        Read the file from the filesystem,
        as either raw text or as a Jinja template,
        and load frontmatter, if any, either way.
        """
        assert "md" in self.data
        if not self.skip_pp:
            logger.debug("... loading markdown from filesystem")
            path = self.config["content_path"]
            pp_env = self.get_pp_env(loader=FrontMatterFSLoader(path))
            self.pp_template = pp_env.get_template(self.data["md"])
            frontmatter = pp_env.loader.fm_map[self.data["md"]]
            merge_dicts(frontmatter, self.data)
            # special case: let frontmatter overwrite default "html" vals
            if PROVIDED_FILENAME_KEY in self.data and "html" in frontmatter:
                self.data["html"] = frontmatter["html"]
            self.twolines = pp_env.loader.twolines[self.data["md"]]
        else:
            logger.info("... reading markdown from file")
            fullpath = os.path.join(self.config["content_path"], self.data["md"])
            with open(fullpath, "r", encoding="utf-8") as f:
                ftext = f.read()
            self.rawtext, frontmatter = parse_frontmatter(ftext)
            merge_dicts(frontmatter, self.data)
            # special case: let frontmatter overwrite default "html" vals
            if PROVIDED_FILENAME_KEY in self.data and "html" in frontmatter:
                self.data["html"] = frontmatter["html"]
            self.twolines = self.rawtext.split("\n", 2)[:2]


    def load_from_generator(self):
        """
        Load the text from a generator function,
        as either raw text or a jinja template.
        Assume no frontmatter in this case.
        """
        if not self.skip_pp:
            pp_env = self.get_pp_env(
                loader=jinja2.DictLoader({"_": self.data["__md_generator"]()}) )
            self.pp_template = pp_env.get_template("_")
        else:
            self.rawtext = self.data["__md_generator"]()

    def provide_default_filename(self):
        """
        Provide a default "html" filename to a page dictionary if one wasn't
        explicitly specified. Frontmatter can overwrite this value, but code
        won't see the frontmatter-provided "html" value when not building the
        target containing that page.
        """
        if "html" in self.data:
            return

        logger.debug("Need to generate html filename for page %s" % self)
        if "md" in self.data:
            # TODO: support other formulas including "tail" or "pretty"
            new_filename = re.sub(r"[.]md$", ".html", self.data["md"])
            if self.config.get("flatten_default_html_paths", True):
                new_filename = new_filename.replace(os.sep, "-")
            self.data["html"] = new_filename
        elif "name" in self.data:
            new_filename = slugify(self.data["name"]).lower()+".html"
            self.data["html"] = new_filename
        else:
            new_filename = str(time.time()).replace(".", "-")+".html"
            self.data["html"] = new_filename
        self.data[PROVIDED_FILENAME_KEY] = True

        logger.debug("Generated html filename '%s' for page: %s" %
                    (new_filename, self))

    def provide_name(self):
        """
        Add the "name" field, if not defined.
        """
        if "name" in self.data:
            return

        if "title" in self.data: # Port over the "title" attribute instead
            self.data["name"] = self.data["title"]
            logger.debug("Guessed page name from title for page %s" % self)
            return
        elif self.twolines:
            logger.debug("Guessing page name from first two lines...")
            try:
                soup = BeautifulSoup(markdown("\n".join(self.twolines)), "html.parser")
                first_h = soup.find(name=re.compile("h[1-6]"))
                self.data["name"] = first_h.get_text()
                logger.debug("... guessed title: '%s'"%self.data["name"])
                return
            except Exception as e:
                traceback.print_tb(e.__traceback__)
                logger.warning(("Couldn't guess title of page '%s' from its "+
                                "first two lines:\n%s") % (self,
                                    "\n".join(self.twolines)))

        if "md" in self.data:
            self.data["name"] = self.data["md"]
            logger.debug("Using placeholder name for page based on md path: '%s'"%self.data["name"])
        else:
            logger.warning("Using a placeholder name for page %s" %
                    str(self.data))
            self.data["name"] = str(time.time()).replace(".", "-")



    def preprocess(self, context):
        try:
            md = self.pp_template.render(**context)
        except Exception as e:
            recoverable_error("Preprocessor error in page %s: %s."%(self, e),
                              self.config.bypass_errors, error=e)
            # Preprocessing failed. Skip this step and reload.
            self.skip_pp = True
            self.load_content()
            return self.md_content(context)
        # Apply markdown-based filters here
        for filter_name in self.filters():
            if "filter_markdown" in dir(self.config.filters[filter_name]):
                logger.debug("... applying markdown filter %s" % filter_name)
                try:
                    md = self.config.filters[filter_name].filter_markdown(
                        md,
                        logger=logger,
                        **context,
                    )
                except Exception as e:
                    recoverable_error("Markdown filter '%s' failed on page %s: %s" %
                                  (filter_name, self, e),
                                  self.config.bypass_errors, error=e)

        logger.info("... markdown is ready")
        self.md = md
        return md

    def md_content(self, context):
        if self.md is not None:
            # return already-preprocessed md
            return self.md
        elif self.rawtext is not None:
            return self.rawtext
        elif not self.skip_pp and self.pp_template is not None:
            return self.preprocess(context)
        else:
            logger.debug("page %s has no rawtext or pp_template"%self.data)
            return ""

    def html_content(self, context, regen=False, save=True):
        """
        Returns the page's contents as HTML. Parses Markdown & runs filters
        if any.
        """
        if self.html is not None and not regen:
            # Reuse saved results if we have them
            return self.html

        md = self.md_content(context)

        if md:
            logger.info("... parsing markdown...")
            html = markdown(md, extensions=["markdown.extensions.extra",
                                            "markdown.extensions.sane_lists"])
        else:
            html = ""

        # Apply raw-HTML-string-based filters here
        for filter_name in self.filters():
            if "filter_html" in dir(self.config.filters[filter_name]):
                logger.debug("... applying HTML filter %s" % filter_name)
                try:
                    html = self.config.filters[filter_name].filter_html(
                            html,
                            logger=logger,
                            **context,
                    )
                except Exception as e:
                    recoverable_error("HTML filter '%s' failed on page %s: %s" %
                            (filter_name, self, e), self.config.bypass_errors,
                            error=e)

        # Some filters would rather operate on a soup than a string.
        # May as well parse once and re-serialize once.
        soup = BeautifulSoup(html, "html.parser")
        self.soup = soup

        # Give each header a unique ID and fill out the Table of Contents
        self.update_toc()

        # Add a "blurb" attribute to the page
        self.provide_blurb()

        # Add this page's plaintext field. ElasticSearch upload uses this.
        self.data["plaintext"] = soup.get_text()

        # Apply soup-based filters here
        for filter_name in self.filters():
            if "filter_soup" in dir(self.config.filters[filter_name]):
                logger.debug("... applying soup filter %s" % filter_name)
                try:
                    self.config.filters[filter_name].filter_soup(
                            soup,
                            logger=logger,
                            **context,
                    )
                    # ^ the soup filters apply to the same object, passed by reference
                except Exception as e:
                    recoverable_error("Soup filter '%s' failed on page %s: %s" %
                                  (filter_name, self, e),
                                  self.config.bypass_errors, error=e)

        logger.info("... re-rendering HTML from soup...")
        html2 = str(soup)
        if save:
            self.html = html2
        return html2

    @staticmethod
    def idify(utext):
        """Make a string ID-friendly (but more unicode-friendly)"""
        utext = re.sub(r'[^\w\s-]', '', utext).strip().lower()
        utext = re.sub(r'[\s-]+', '-', utext)
        if not len(utext):
            # Headers must be non-empty
            return '_'
        return utext

    def update_toc(self):
        """
        Assign unique IDs to header elements in a BeautifulSoup object, and
        update internal table of contents accordingly.
        The resulting ToC is a list of objects, each in the form:
        {
            "text": "Header Content as Text",
            "id": "header-content-as-text", # doesn't have the # prefix
            "level": 1, #1-6, based on h1, h2, etc.
        }
        Also add the "headermap" field used for ES JSON, which is in the form
        {
            "Header Content as Text": "#header-content-as-text"
        }
        """

        self.toc = []
        uniqIDs = {}
        headermap = {}
        headers = self.soup.find_all(name=re.compile("h[1-6]"))
        for h in headers:
            h_id = self.idify(h.get_text())
            if h_id not in uniqIDs.keys():
                uniqIDs[h_id] = 0
            else:
                # not unique, append -1, -2, etc. to this instance
                uniqIDs[h_id] += 1
                h_id = "{id}-{n}".format(id=h_id, n=uniqIDs[h_id])

            h["id"] = h_id
            self.toc.append({
                "text": h.get_text(),
                "id": h_id,
                "level": int(h.name[1])
            })
            # ElasticSearch doesn't like dots in keys, so escape those
            escaped_name = h.get_text().replace(".","-")
            headermap[escaped_name] = "#"+h_id
        self.data["headermap"] = headermap

    def legacy_toc(self):
        """
        Return an HTML table of contents in the legacy format from the internal
        table of contents list.
        """
        soup = BeautifulSoup("", "html.parser")
        for h in self.toc:
            if h["level"] > 3:
                # legacy toc only goes down to h3
                continue
            a = soup.new_tag("a", href="#"+h["id"])
            a.string = h["text"]
            li = soup.new_tag("li")
            li["class"] = "level-{n}".format(n=h["level"])
            li.append(a)
            soup.append(li)
            soup.append("\n")
        return str(soup)

    def provide_blurb(self):
        """
        Add a "blurb" field, based on the first paragraph in the page, if one
        is not already provided.
        """
        if "blurb" in self.data:
            return
        p = self.soup.find("p")
        while p:
            if p.get_text().strip():
                self.data["blurb"] = p.get_text()
                break
            else:
                p = p.find_next_sibling("p")
        else:
            logger.debug("Couldn't find a paragraph with text in page %s." % self.data["name"])
            # No text? No blurb.
            self.data["blurb"] = ""

    def render(self, use_template, context):
        """
        Render the entire page using the given template & context.
        """
        # TODO: try block around html_content()?
        html_content = self.html_content(context)

        out_html = use_template.render(
            content=html_content,
            sidebar_content=self.legacy_toc(),
            page_toc=self.legacy_toc(),
            headers=self.toc,
            **context,
        )
        return out_html

    def es_json(self, use_template, context):
        """
        Return JSON for uploading to ElasticSearch
        """
        # Parse HTML to get the required "bonus" fields like blurb and toc, but
        # don't save the "es"-rendered HTML as the final output, in case this
        # is a build-HTML-and-upload-ES run.
        es_context = {**context}
        es_context["mode"] = "es"
        self.html_content(es_context, save=False)

        # Set up Jinja env for rendering strings
        es_env = self.get_pp_env(loader=None)#TODO: does this even work?

        def eval_es_string(expr):
            try:
                result = eval(expr, {}, context)
            except Exception as e:
                recoverable_error(
                    "__dactyl_eval__ failed on expression '{expr}': {e}".format(
                        expr=expr, e=e),
                    self.config.bypass_errors
                )
                result = expr
            return result

        def render_es_field(value):
            if type(value) == str: # jinja-render strings
                field_templ = es_env.from_string(value)
                try:
                    parsed_field = field_templ.render(context)
                except jinja2.exceptions.TemplateSyntaxError as e:
                    recoverable_error(
                        "Couldn't parse value '{value}' in ES template: {e}".format(
                            value, e),
                        self.config.bypass_errors
                    )
                return parsed_field
            elif type(value) in (type(None), int, float, bool): # preserve literals
                return value
            elif type(value) == dict and ES_EVAL_KEY in value.keys():
                return eval_es_string(value[ES_EVAL_KEY])
            elif type(value) == dict: # recurse!
                return {rkey: render_es_field(rval) for rkey,rval in value.items()}
            elif type(value) == list: # recurse!
                return [render_es_field(rval) for rval in value]
            else:
                recoverable_error("Unknown type '{t}' in ES template" % type(value))
                return ""

        wout = {key: render_es_field(val) for key,val in use_template.items()}
        return json.dumps(wout, indent=4, separators=(',', ': '))

    def filepath(self, mode):
        """
        Returns the preferred filename to write output to based on the provided
        mode.
        """

        if mode == "es":
            # use .json as file extension instead
            fp = re.sub(r'(.+)\.html?$', r'\1.json', self.data["html"], flags=re.I)
            if fp[-5:] != ".json": # substitution didn't work
                fp = fp+".json"
            return fp
        elif mode == "md":
            if "md" in self.data:
                # reuse the input .md filename as output
                fp = self.data["md"]
                if ":" in fp: # http: or https: probably
                    fp = slugify(f)
            else:
                # use the html field, but change the file extension to .md
                fp = re.sub(r'(.+)\.html?$', r'\1.md', self.data["html"], flags=re.I)
                if fp[-3:] != ".md": # substitution didn't work
                    fp = fp+".md"
            return fp
        else:
            # for pdf or html just use the html field as-is
            return self.data["html"]

    def gain_filters(self, filterlist):
        """
        Called by target to add its filters to this page's.
        """
        if "filters" in self.data:
            self.data["filters"] = filterlist + self.data["filters"]
        else:
            self.data["filters"] = filterlist

    def filters(self, save=True):
        """
        Returns the names of filters to use when processing this page.
        """
        if self.ffp:
            # Return saved results
            return self.ffp

        # Create de-duplicated list of loaded filters only
        ffp = []
        for f in self.config["default_filters"]:
            if f not in ffp and f in self.config.filters:
                ffp.append(f)
        for f in self.data.get("filters", [])    :
            if f not in ffp and f in self.config.filters:
                ffp.append(f)

        logger.debug("...filters for page: %s"%ffp)
        if save:
            self.ffp = ffp
        return ffp

    def __repr__(self):
        if "name" in self.data and "html" in self.data:
            return "<Page '{name}' ({html})>".format(
                name=self.data["name"],
                html=self.data["html"],
            )
        if "name" in self.data:
            return "<Page '{name}'>".format(name=self.data["name"])
        if "html" in self.data:
            return "<Page ({html})>".format(html=self.data["html"])
        strdata = str(self.data)
        if len(strdata) > 30:
            strdata = strdata[:27]+"..."
        return "<Page {{{strdata}}}>".format(strdata=strdata)
