# -*- coding: utf-8 -*-
from __future__ import unicode_literals, print_function

from collections import defaultdict
from datetime import datetime
from tempfile import mkstemp
from io import BytesIO
from wsgiref.handlers import format_date_time as format_http_date
from wsgiref.util import request_uri
import codecs
import logging
import logging.handlers
import os
import re
import time
import calendar
import filecmp
import socket
from ferenda.compat import OrderedDict

# 3rd party
import pkg_resources
from lxml import etree
from lxml.builder import ElementMaker
import lxml.html

from rdflib import Graph, Literal, Namespace, URIRef, RDF
import bs4
import requests
import requests.exceptions

from six import text_type as str
from six import binary_type as bytes

from six.moves.urllib_parse import quote

# mine
import ferenda
from ferenda import util, errors, decorators
from ferenda import (Describer, LayeredConfig, TripleStore, FulltextIndex,
                     Document, DocumentEntry, NewsCriteria, TocCriteria,
                     TocPageset, TocPage, DocumentStore, Transformer)
from ferenda.elements import (AbstractElement, serialize, Body, Nav, Link,
                              Section, Subsection, Subsubsection, Heading,
                              UnorderedList, ListItem, Preformatted, Paragraph)
from ferenda.elements.html import elements_from_soup
from ferenda.thirdparty import patch, httpheader
# establish two central RDF Namespaces at the top level
DCT = Namespace(util.ns['dct'])
PROV = Namespace(util.ns['prov'])

class DocumentRepository(object):

    """Base class for downloading, parsing and generating HTML versions of a repository of documents.

    Start building your application by subclassing this class, and
    then override methods in order to customize the downloading,
    parsing and generation behaviour.

    :param \*\*kwargs: Any named argument overrides any
                   similarly-named configuration file parameter.
    
    Example:
    
    >>> class MyRepo(DocumentRepository):
    ...     alias="myrepo"
    ...
    >>> d = MyRepo(datadir="/tmp/ferenda")
    >>> d.store.downloaded_path("mybasefile").replace(os.sep,'/') == '/tmp/ferenda/myrepo/downloaded/mybasefile.html'
    True

    .. note::

       This class has a ridiculous amount of methods that you can
       override to control most of ferendas behaviour in all
       stages. For basic usage, you need only a fraction of
       them. Please don't be intimidated/horrified.

    """

#    There are seven main entry points into the module, with the
#    following principal call chains:
#
#    download
#        download_get_basefiles
#            download_single
#                downloaded_path
#                download_if_needed
#                remote_url
#                download_update_entry
#    parse
#        parsed_path
#        soup_from_basefile
#        parse_from_soup
#        render_xhtml
#
#    relate
#
#    generate
#        generated_file
#        prep_annotation_file
#            graph_to_annotation_file
#
#    toc
#        toc_query
#        toc_criteria
#            toc_predicates
#        toc_item
#
#    news
#        news_selections
#        news_selection
#            news_get_entry
#
#    frontpage_content

    #
    # general class properties
    # FIXME: Duplicated in documentstore -- how do we unify?
    downloaded_suffix = ".html"
    """File suffix for the main document format. Determines the suffix
    of downloaded files."""

    # FIXME: Duplicated in documentstore -- how do we unify?
    storage_policy = "file"
    """Some repositories have documents in several formats, documents
    split amongst several files or embedded resources. If
    ``storage_policy`` is set to ``dir``, then each document gets its own
    directory (the default filename being ``index`` +suffix),
    otherwise each doc gets stored as a file in a directory with other
    files.  Affects
    :py:meth:`ferenda.DocumentStore.path` (and therefore
    all other ``*_path`` methods)"""

    alias = "base"
    """A short name for the class, used by the command line
    ``ferenda-build.py`` tool. Also determines where to store
    downloaded, parsed and generated files. When you subclass
    :py:class:`~ferenda.DocumentRepository` you *must* override
    this."""

    namespaces = ['rdf', 'rdfs', 'xsd', 'xsi', 'dct', 'skos', 'foaf', 'xhv', 'owl', 'prov', 'bibo']
    """The namespaces that are included in the XHTML and RDF files
    generated by :py:meth:`~ferenda.DocumentRepository.parse`. This
    can be a list of strings, in which case the strings are assumed to
    be well-known prefixes to established namespaces, or a list of
    (prefix, namespace) tuples. All well-known prefixes are available
    in :py:data:`ferenda.util.ns`."""

    required_predicates = [RDF.type]
    """A list of RDF predicates that should be present in the outdata. If
    any of these are missing, a warning is logged.

    """

    #
    # download() related class properties

    start_url = "http://example.org/"

    """The main entry page for the remote web store of documents. May
    be a list of documents, a search form or whatever. If it's
    something more complicated than a simple list of documents, you
    need to override :py:meth:`~ferenda.DocumentRepository.download`
    in order to tell which documents are to be downloaded."""

    document_url_template = "http://example.org/docs/%(basefile)s.html"
    """A string template for creating URLs for individual documents on
    the remote web server. Directly used by
    :py:meth:`~ferenda.DocumentRepository.remote_url` and indirectly
    by :py:meth:`~ferenda.DocumentRepository.download_single`."""

    document_url_regex = "http://example.org/docs/(?P<basefile>\w+).html"
    """A regex that matches URLs for individual documents -- the
    reverse of what
    :py:data:`~ferenda.DocumentRepository.document_url_template` is
    used for. Used by
    :py:meth:`~ferenda.DocumentRepository.download()` to find suitable
    links if :py:data:`~ferenda.DocumentRepository.basefile_regex`
    doesn't match. Must define the named group ``basefile`` using the
    ``(?P<basefile>...)`` syntax"""

    # matches "ID: foo/123" or "ID: Bar:Baz/Quux" but not "ID: Foo bar"
    basefile_regex = "^ID: ?(?P<basefile>[\w\d\:\/]+)$"
    """A regex for matching document names in link text, as used by
    :py:meth:`~ferenda.DocumentRepository.download()`. Must define a
    named group ``basefile``, just like
    :py:data:`~ferenda.DocumentRepository.document_url_template`."""

    #
    # parse() specific class properties
    rdf_type = Namespace(util.ns['foaf']).Document
    """The RDF type of the documents you are handling (expressed as a
    :py:class:`rdflib.term.URIRef` object)."""

    source_encoding = "utf-8"
    """The character set that the source HTML documents use (if
    applicable)."""

    lang = "en"
    """The language which the source documents are assumed to be
    written in (unless otherwise specified), and the language which
    output document should use."""

    # css selectors, handled by BeautifulSoup's select() method
    parse_content_selector = "body"
    """CSS selector used to select the main part of the document
    content by the default
    :py:meth:`~ferenda.DocumentRepository.parse` implementation."""

    parse_filter_selectors = ["script"]
    """CSS selectors used to filter/remove certain parts of the
    document content by the default
    :py:meth:`~ferenda.DocumentRepository.parse` implementation."""

    #
    # generate() specific class properties
    xslt_template = "res/xsl/generic.xsl"
    """A template used by
    :py:meth:`~ferenda.DocumentRepository.generate` to transform the
    XML file into browser-ready HTML. If your document type is
    complex, you might want to override this (and write your own XSLT
    transform). You should include ``base.xslt`` in that template,
    though."""

    sparql_annotations = "res/sparql/annotations.rq"
    """A template SPARQL CONSTRUCT query for document annotations."""

    # FIXME: Sphinx really wants to treat this class as a reference,
    # but cannot resolve it
    documentstore_class = DocumentStore
#    """Class that implements the :class:`~ferenda.DocumentStore` interface."""

    def __init__(self, **kwargs):
        """See :py:class:`~ferenda.DocumentRepository`."""
        codedefaults = self.get_default_options()
        defaults = util.merge_dict_recursive(codedefaults, kwargs)
        self._config = LayeredConfig(defaults=defaults)
        if not hasattr(self, 'store'):
            self.store = self.documentstore_class(self.config.datadir + os.sep + self.alias)
        # should documentstore have a connection to self, ie
        # self.store = DocumentStore(basedir, self) ?
        self.store.downloaded_suffix = self.downloaded_suffix
        self.store.storage_policy = self.storage_policy

        logname = self.alias
        # alternatively (nonambigious and helpful for debugging, but verbose)
        # logname = self.__class__.__module__+"."+self.__class__.__name__
        self.log = self._setup_logger(logname)

        self.ns = {}
        for ns in self.namespaces:
            if isinstance(ns, tuple):
                prefix, uri = ns
                self.ns[prefix] = Namespace(uri)
            else:
                prefix = ns
                # assume that any standalone prefix is well known
                self.ns[prefix] = Namespace(util.ns[prefix])

    @property
    def config(self):
        return self._config

    @config.setter
    def config(self, config):
        self._config = config
        self.store = self.documentstore_class(
            config.datadir + os.sep + self.alias,
            downloaded_suffix=self.downloaded_suffix,
            storage_policy=self.storage_policy)


    def get_default_options(self):
        """Returns the class' configuration default configuration
        properties. These can be overridden by a configution file, or
        by named arguments to
        :py:meth:`~ferenda.DocumentRepository.__init__`. See
        :ref:`configuration` for a list of standard configuration
        properties (your subclass is free to define and use additional
        configuration properties).

        :returns: default configuration properties
        :rtype: dict
        """

        return {  # 'loglevel': 'INFO',
            'datadir': 'data',
            'patchdir': 'patches',
            'force': False,
            'parseforce': False,
            'serializejson': False,
            'compress': "", # don't compress by default
            'generateforce': False,
            'fsmdebug': False,
            'refresh': False,
            'lastdownload': datetime,
            'downloadmax': int,
            'conditionalget': True,
            'url': 'http://localhost:8000/',
            'fulltextindex': True,
            'useragent': 'ferenda-bot',
            'storetype': 'SQLITE',
            'storelocation': 'data/ferenda.sqlite',
            'storerepository': 'ferenda',
            'indextype': 'WHOOSH',
            'indexlocation': 'data/whooshindex',
            'republishsource': False,
            'combineresources': False,
            'cssfiles': ['http://fonts.googleapis.com/css?family=Raleway:200,100',
                             'res/css/normalize-1.1.3.css',
                             'res/css/main.css',
                             'res/css/ferenda.css'],
            'jsfiles': ['res/js/jquery-1.10.2.js',
                        'res/js/modernizr-2.6.3.js',
                        'res/js/respond-1.3.0.js',
                        'res/js/ferenda.js'],
            'staticsite': False,
            'class': self.qualified_class_name()
        }

    @classmethod
    def setup(cls, action, config):
        """Runs before any of the ``*_all`` methods starts executing. It just
calls the appropriate setup method, ie if *action* is ``parse``, then
this method calls ``parse_all_setup`` (if defined) with the *config*
object as single parameter."""

        if hasattr(cls, action + "_all_setup"):
            cbl = getattr(cls, action + "_all_setup")
            if callable(cbl):
                return cbl(config)

    @classmethod
    def teardown(cls, action, config):
        """Runs after any of the ``*_all`` methods has finished executing. It
just calls the appropriate teardown method, ie if *action* is
``parse``, then this method calls ``parse_all_teardown`` (if defined)
with the *config* object as single parameter.

        """

        if hasattr(cls, action + "_all_teardown"):
            cbl = getattr(cls, action + "_all_teardown")
            if callable(cbl):
                return cbl(config)

    def get_archive_version(self, basefile):
        """Get a version identifier for the current version of the
        document identified by ``basefile``.

        The default implementation simply increments most recent
        archived version identifier, starting at "1". If versions in
        your docrepo are normally identified in some other way (such
        as SCM revision numbers, dates or similar) you should override
        this method to return those identifiers.

        :param basefile: The basefile of the document to archive
        :type basefile: str
        :returns: The version identifier for the current version of
                  the document.
        :rtype:   str
        """
        return str(len(list(self.store.list_versions(basefile))) + 1)

    def qualified_class_name(self):
        """The qualified class name of this class

        :returns: class name (e.g. ``ferenda.DocumentRepository``)
        :rtype:   str
        """
        return self.__class__.__module__ + "." + self.__class__.__name__

    def canonical_uri(self, basefile):
        """The canonical URI for the document identified by ``basefile``.

        :returns: The canonical URI
        :rtype: str
        """
        # Note that there might not be a 1:1 mappning between
        # documents/basefiles and URIs -- don't know what we should do
        # in those cases.
        #
        # It might also be impossible to provide the canonical_uri
        # without actually parse()ing the document
        return "%sres/%s/%s" % (self.config.url, self.alias, basefile)

    def dataset_uri(self, param=None, value=None):
        """Returns the URI that identifies the dataset that this docrepository
        provides. The default implementation is based on the url
        config parameter and the alias attribute of the class,
        c.f. ``http://localhost:8000/dataset/base``.

        :param param: An optional parameter name represeting a way of createing a subset of the dataset (eg. all document whose title starts with a particular letter)
        :param value: A value for *param* (eg. "a")

        >>> d = DocumentRepository()
        >>> d.alias == 'base'
        True
        >>> d.config.url = "http://example.org/"
        >>> d.dataset_uri() == 'http://example.org/dataset/base'
        True
        >>> d.dataset_uri("title","a") == 'http://example.org/dataset/base?title=a'
        True
        """

        uri = "%sdataset/%s" % (self.config.url, self.alias)
        if param and value:
            uri += "?%s=%s" % (param, quote(value))
        return uri

    def basefile_from_uri(self, uri):
        """The reverse of
           :meth:`~ferenda.DocumentRepository.canonical_uri`. Returns
           None if the uri doesn't map to a basefile in this repo.

        >>> d = DocumentRepository()
        >>> d.alias == "base"
        True
        >>> d.config.url = "http://example.org/"
        >>> d.basefile_from_uri("http://example.org/res/base/123/a") == "123/a"
        True
        >>> d.basefile_from_uri("http://example.org/res/base/123/a#S1") == "123/a"
        True
        >>> d.basefile_from_uri("http://example.org/res/other/123/a") # None
        
        """
        if uri.startswith(self.config.url + "res/"):
            path = uri[len(self.config.url + "res/"):]
            if "/" in path:
                alias, basefile = path.split("/", 1)
                if "#" in basefile:
                    basefile = basefile.split("#")[0]
                if alias == self.alias:
                    return basefile

    def dataset_params_from_uri(self, uri):
        """Given a parametrized dataset URI, return the parameter and value
           used (or an empty tuple, if it is a dataset URI handled by
           this repo, but without any parameters).

        >>> d = DocumentRepository()
        >>> d.alias == 'base'
        True
        >>> d.config.url = "http://example.org/"
        >>> d.dataset_params_from_uri("http://example.org/dataset/base?title=a") == ('title', 'a')
        True
        >>> d.dataset_params_from_uri("http://example.org/dataset/base") == ()
        True

        """

        wantedprefix = self.config.url + "dataset/" + self.alias
        if uri == wantedprefix or ("?" in uri and
                                   uri.startswith(wantedprefix)):
            path = uri[len(wantedprefix) + 1:]
            if "=" in path:
                return tuple(path.split("=", 1))
            else:
                return ()

    #
    #
    # STEP 1: Download documents from the web
    #
    #
    @decorators.action
    @decorators.recordlastdownload
    def download(self, basefile=None):
        """Downloads all documents from a remote web service.

        The default generic implementation assumes that all documents
        are linked from a single page (which has the url of
        :py:data:`~ferenda.DocumentRepository.start_url`), that they
        all have URLs matching the
        :py:data:`~ferenda.DocumentRepository.document_url_regex` or
        that the link text is always equal to basefile (as determined
        by :py:data:`~ferenda.DocumentRepository.basefile_regex`). If
        these assumptions don't hold, you need to override this
        method.

        If you do override it, your download method should read and set the
        ``lastdownload`` parameter to either the datetime of the last
        download or any other module-specific string (id number or
        similar).

        You should also read the ``refresh`` parameter. If it is
        ``True`` (the default), then you should call
        :py:meth:`~ferenda.DocumentRepository.download_single` for
        every basefile you encounter, even though they may already
        exist in some form on
        disk. :py:meth:`~ferenda.DocumentRepository.download_single`
        will normally be using conditional GET to see if there is a
        newer version available.

        See :ref:`implementing-download` for more details.

        :returns: True if any document was downloaded, False otherwise.
        :rtype: bool
        """
        if basefile:
            if self.document_url_template:
                return self.download_single(basefile)
            else:
                raise ValueError(
                    "Downloading single basefile '%s' not supported (no way to convert basefile to url)" % basefile)
        lastdownload = self.config.lastdownload
        if lastdownload:
            self.log.debug("download: Last download was at %s" % lastdownload)
        else:
            self.log.debug("download: Starting full download")
        # NOTE: This very generic implementation of download has no
        # use for lastdownload, as all the documents it can find are
        # the one linked from the start page. Therefore it's not used
        # for anything else than a diagnostic tool.

        refresh = self.config.refresh
        if refresh:
            self.log.debug("download: Refreshing all downloaded files")
        else:
            self.log.debug("download: Not re-downloading downloaded files")

        self.log.debug("Starting at %s" % self.start_url)
        updated = False

        resp = requests.get(self.start_url)
        tree = lxml.html.document_fromstring(resp.text)
        tree.make_links_absolute(self.start_url, resolve_base_href=True)
        for (basefile, link) in self.download_get_basefiles(tree.iterlinks()):
            if (refresh or
                    (not os.path.exists(self.store.downloaded_path(basefile)))):
                ret = self.download_single(basefile, link)
                updated = updated or ret
        self.config.lastdownload = datetime.now()
        return updated

    @decorators.downloadmax
    def download_get_basefiles(self, source):
        """Given *source* (a iterator that provides (element, attribute, link,
        pos) tuples, like ``lxml.etree.iterlinks()``), generate tuples
        (basefile, link) for all document links found in *source*.

        """

        for (element, attribute, link, pos) in source:
            url = basefile = None

            # Two step process: First examine link text to see if
            # basefile_regex match. If not, examine link url to see
            # if document_url_regex
            if (self.basefile_regex and
                element.text and
                    re.search(self.basefile_regex, element.text)):
                m = re.search(self.basefile_regex, element.text)
                yield(m.group("basefile"), link)
            elif self.document_url_regex and re.match(self.document_url_regex, link):
                m = re.match(self.document_url_regex, link)
                if m:
                    yield(m.group("basefile"), link)

    def download_single(self, basefile, url=None):
        """Downloads the document from the web (unless explicitly
        specified, the URL to download is determined by
        :py:data:`~ferenda.DocumentRepository.document_url_template` combined
        with basefile, the location on disk is determined by the
        function
        :py:meth:`~ferenda.DocumentStore.downloaded_path`).

        If the document exists on disk, but the version on the web is
        unchanged (determined using a conditional GET), the file on disk
        is left unchanged (i.e. the timestamp is not modified).

        :param basefile: The basefile of the document to download
        :type basefile: string
        :param url: The URL to download (optional)
        :type url: str
        :returns: ``True`` if the document was downloaded and stored on
                  disk, ``False`` if the file on disk was not updated.
        """
        if url is None:
            url = self.remote_url(basefile)

        updated = False
        created = False

        filename = self.store.downloaded_path(basefile)
        created = not os.path.exists(filename)
        # util.print_open_fds()
        if self.download_if_needed(url, basefile):
            if created:
                self.log.info("%s: downloaded from %s" % (basefile, url))
            else:
                self.log.info(
                    "%s: downloaded new version from %s" % (basefile, url))
            updated = True
        else:
            self.log.debug("%s: exists and is unchanged" % basefile)

        entry = DocumentEntry(self.store.documententry_path(basefile))
        now = datetime.now()
        entry.orig_url = url
        if created:
            entry.orig_created = now
        if updated:
            entry.orig_updated = now
        entry.orig_checked = now
        entry.save()

        return updated

    def _addheaders(self, filename=None):
        headers = {"User-agent": self.config.useragent}
        if filename:
            # we set both if-none-match and if-modified-since if we
            # can. We've encountered at least one server which sends
            # ETags but don't return 304 when the appropriate ETag is
            # returned in a if-none-match header (but return 304 when
            # if-modified-since is used)
            if os.path.exists(filename + ".etag"):
                headers["If-none-match"] = util.readfile(filename + ".etag")
            if os.path.exists(filename):
                stamp = os.stat(filename).st_mtime
                headers["If-modified-since"] = format_http_date(stamp)
        return headers

    def download_if_needed(self, url, basefile, archive=True, filename=None, sleep=1):
        """Downloads a remote resource to a local file. If a different
        version is already in place, archive that old version.

        :param      url: The url to download
        :type       url: str
        :param basefile: The basefile of the document to download
        :type  basefile: str
        :param  archive: Whether to archive existing older versions of
                         the document, or just delete the previously
                         downloaded file.
        :type   archive: bool
        :param filename: The filename to download to. If not provided,
                         the filename is derived from the supplied
                         basefile
        :type  filename: str
        :returns:        True if the local file was updated (and archived),
                         False otherwise.
        :rtype:          bool

        """
        if not filename:
            filename = self.store.downloaded_path(basefile)
        if self.config.conditionalget:
            # sets if-none-match and/or if-modified-since headers
            headers = self._addheaders(filename)
        else:
            headers = self._addheaders()

        fileno, tmpfile = mkstemp()
        fp = os.fdopen(fileno)
        fp.close()

        # Since this part, containing the actual HTTP request call, is
        # called repeatedly, we take extra precautions in the event of
        # temporary network failures etc. Try 5 times with 1 second
        # pause inbetween before giving up.
        fetched = False
        remaining_attempts = 5
        try:
            while (not fetched) and (remaining_attempts > 0):
                try:
                    response = requests.get(url, headers=headers, timeout=10)
                    fetched = True
                # socket.timeout ought to be caught by requests and
                # repackaged as requests.exceptions.Timeout, but in
                # one case it wasn't
                except (requests.exceptions.ConnectionError,
                        requests.exceptions.Timeout,
                        socket.timeout) as e:
                    self.log.warning(
                        "Failed to fetch %s: error %s (%s remaining attempts)" % (url, e, remaining_attempts))
                    remaining_attempts -= 1
                    time.sleep(sleep)

            if not fetched:
                self.log.error("Failed to fetch %s, giving up" % url)
                return False
        # handles other errors except ConnectionError
        except requests.exceptions.RequestException as e:
            self.log.error("Failed to fetch %s: error %s" % (url, e))
            raise e
        if response.status_code == 304:
            self.log.debug("%s: 304 Not modified" % url)
            return False  # ie not updated
        elif response.status_code > 400:
            response.raise_for_status()

        with open(tmpfile, "wb") as fp:
            fp.write(response.content)

        if not os.path.exists(filename):
            util.robust_rename(tmpfile, filename)
            updated = True
        elif self.download_is_different(filename, tmpfile):
            if archive:
                version = self.get_archive_version(basefile)
                self.store.archive(basefile, version)
            util.robust_rename(tmpfile, filename)
            updated = True
        else:
            updated = False

        if updated:
            # OK we have a new file in place. Now examine the
            # headers to find if we should change file
            # modification time (last-modified) and/or create a
            # .etag file (etag)
            if response.headers.get("last-modified"):
                mtime = calendar.timegm(util.parse_rfc822_date(
                    response.headers["last-modified"]).timetuple())
                os.utime(filename, (time.time(), mtime))
            if response.headers.get("etag"):
                with open(filename + ".etag", "w") as fp:
                    fp.write(response.headers["etag"])
        return updated

    def download_is_different(self, existing, new):
        """Returns True if the new file is semantically different from the existing file."""
        return not filecmp.cmp(new, existing, shallow=False)


    def remote_url(self, basefile):
        """Get the URL of the source document at it's remote location,
        unless the source document is fetched by other means or if it
        cannot be computed from basefile only. The default
        implementation uses
        :py:data:`~ferenda.DocumentRepository.document_url_template`
        to calculate the url.

        Example:

        >>> d = DocumentRepository()
        >>> d.remote_url("123/a") == 'http://example.org/docs/123/a.html'
        True
        >>> d.document_url_template = "http://mysite.org/archive/%(basefile)s/"
        >>> d.remote_url("123/a") == 'http://mysite.org/archive/123/a/'
        True
        
        :param basefile: The basefile of the source document
        :type basefile: str
        :returns: The remote url where the document can be fetched, or ``None``.
        :rtype: str
        """
        return self.document_url_template % {'basefile': quote(basefile)}

    def generic_url(self, basefile, maindir, suffix):
        """
        Analogous to
        :py:meth:`ferenda.DocumentStore.path`, calculate
        the full local url for the given basefile and stage of
        processing.

        :param basefile: The basefile for which to calculate the local url
        :type  basefile: str
        :param  maindir: The processing stage directory (normally
                         ``downloaded``, ``parsed``, or ``generated``)
        :type   maindir: str
        :param   suffix: The file extension including period (i.e. ``.txt``,
                         not ``txt``)
        :type    suffix: str
        :returns: The local url
        :rtype: str
        """
        path = "%s/%s/%s%s" % (self.alias, maindir, basefile, suffix)
        return self.config.url + path

    def downloaded_url(self, basefile):
        """Get the full local url for the downloaded file for the
        given basefile.
        
        :param basefile: The basefile for which to calculate the local url
        :type  basefile: str
        :returns: The local url
        :rtype: str

        >>> d = DocumentRepository()
        >>> d.downloaded_url("123/a") == "http://localhost:8000/base/downloaded/123/a.html"
        True
        """

        return self.generic_url(basefile, 'downloaded', self.downloaded_suffix)

    #
    #
    # STEP 2: Parse the downloaded data into a structured XML document
    # with RDFa metadata.
    #
    #
    @classmethod
    def parse_all_setup(cls, config):
        """
        Runs any action needed prior to parsing all documents in a
        docrepo. The default implementation does nothing.

        .. note::

           This is a classmethod for now (and that's why a config
           object is passsed as an argument), but might change to a
           instance method.
        """

    @classmethod
    def parse_all_teardown(cls, config):
        """ 
        Runs any cleanup action needed after parsing all documents in
        a docrepo. The default implementation does nothing.

        .. note::

           Like :py:meth:`~ferenda.DocumentRepository.parse_all_setup`
           this might change to a instance method.
        """

    def parseneeded(self, basefile):
        """Returns True iff there is a need to parse the given basefile. If the resulting parsed file exists and is newer than the downloaded file, there is typically no reason to parse the file."""
        infile = self.store.downloaded_path(basefile)
        outfile = self.store.parsed_path(basefile)
        return not util.outfile_is_newer([infile], outfile)
        
    @decorators.action
    @decorators.managedparsing
    def parse(self, doc):
        """Parse downloaded documents into structured XML and RDF.
       
        It will also save the same RDF statements in a separate
        RDF/XML file.

        You will need to provide your own parsing logic, but often
        it's easier to just override parse_from_soup (assuming your
        indata is in a HTML format parseable by BeautifulSoup) and let
        the base class read and write the files.

        If your data is not in a HTML format, or BeautifulSoup is not
        an appropriate parser to use, override this method.

        :param doc: The document object to fill in.
        :type  doc: ferenda.Document
        """
        soup = self.soup_from_basefile(doc.basefile, self.source_encoding)
        self.parse_metadata_from_soup(soup, doc)
        self.parse_document_from_soup(soup, doc)
        return True  # Signals that everything is OK

    def soup_from_basefile(self, basefile, encoding='utf-8', parser='lxml'):
        """
        Load the downloaded document for basefile into a BeautifulSoup object

        :param basefile: The basefile for the downloaded document to parse
        :type  basefile: str
        :param encoding: The encoding of the downloaded document
        :type  encoding: str 
        :returns: The parsed document as a ``BeautifulSoup`` object

        .. note::

           Helper function. You probably don't need to override it.
        """
        filename = self.store.downloaded_path(basefile)
        if not os.path.exists(filename):
            raise errors.NoDownloadedFileError("File '%s' not found" % filename)
        with codecs.open(filename, encoding=encoding, errors='replace') as fp:
            soup = bs4.BeautifulSoup(fp.read(), parser)
        return soup

    def parse_metadata_from_soup(self, soup, doc):
        """
        Given a BeautifulSoup document, retrieve all document-level
        metadata from it and put it into the given ``doc`` object's
        ``meta`` property.

        .. note::

           The default implementation sets ``rdf:type``,
           ``dct:title``, ``dct:identifier`` and
           ``prov:wasGeneratedBy`` properties in ``doc.meta``, as well
           as setting the language of the document in ``doc.lang``.
        
        :param soup: A parsed document, as ``BeautifulSoup`` object
        :param  doc: Our document
        :type   doc: ferenda.Document
        :returns: None
        """
        # set rdf:type and dct:identifier of document automatically?
        # set title and other simple things
        # Default language unless we can find out from source doc?
        # Check html/@xml:lang || html/@lang
        root = soup.find('html')
        try:
            doc.lang = root['xml:lang']
        except (KeyError, TypeError):
            try:
                doc.lang = root['lang']
            except (KeyError, TypeError):
                doc.lang = self.lang
        try:
            title = soup.find('title').string
        except AttributeError:
            title = None
        # create document-level metadata
        d = Describer(doc.meta, doc.uri)
        d.rdftype(self.rdf_type)
        if title:
            d.value(self.ns['dct'].title, Literal(title, lang=doc.lang))
        d.value(self.ns['dct'].identifier, doc.basefile)
        d.value(self.ns['prov'].wasGeneratedBy, self.qualified_class_name())

    def parse_document_from_soup(self, soup, doc):
        """
        Given a BeautifulSoup document, convert it into the provided
        ``doc`` object's ``body`` property as suitable
        :py:mod:`ferenda.elements` objects.

        .. note::

           The default implementation respects
           :py:data:`~ferenda.DocumentRepository.parse_content_selector`
           and
           :py:data:`~ferenda.DocumentRepository.parse_filter_selectors`.

        :param soup: A parsed document as a ``BeautifulSoup`` object
        :param  doc: Our document
        :type   doc: ferenda.Document
        :returns: None
        """

        soups = soup.select(self.parse_content_selector)
        if len(soups) == 0:
            raise errors.ParseError("%s: parse_content_selector %r matches nothing" %
                                   (doc.basefile, self.parse_content_selector))
        if len(soups) > 1:
            self.log.warn("%s: parse_content_selector %r matches more than one tag" %
                          (doc.basefile, self.parse_content_selector))
        soup = soups[0]
        for filter_selector in self.parse_filter_selectors:
            for tag in soup.select(filter_selector):
                # tag.decompose()
                tag.extract()  # decompose fails on some trees

        doc.body = elements_from_soup(soup)

    def patch_if_needed(self, basefile, text):
        """Given *basefile* and the entire *text* of the downloaded or
        intermediate document, find if there exists a patch file under
        ``self.config.patchdir``, and if so, applies it. Returns
        (patchedtext, patchdescription) if so, (text,None)
        otherwise.

        """

        # 1. do we have a patch?
        patchstore = self.documentstore_class(self.config.patchdir + os.sep + self.alias)
        patchpath = patchstore.path(basefile, "patches", ".patch")
        descpath = patchstore.path(basefile, "patches", ".desc")
        if os.path.exists(patchpath):
            # 4. make sure error msgs from the patch modules are
            # available if we fail.
            from io import StringIO
            pbuf = StringIO()
            plog = logging.getLogger('ferenda.thirdparty.patch')
            plog.setLevel(logging.WARNING)
            plog.addHandler(logging.StreamHandler(pbuf))

            # 2. read and parse it
            with open(patchpath) as fp:
                ps = patch.PatchSet()
                success = ps.parse(fp)
            if not success:
                raise errors.PatchError(
                    "Patch %s couldn't be parsed: %s" % (patchpath, pbuf.getvalue()))
            assert len(ps.items) == 1
            # 3. Create a temporary file with the file to be patched
            # open tmpfile
            fileno, tmpfile = mkstemp()
            fp = os.fdopen(fileno, "wb")
            # dump text to tmpfile
            fp.write(text.encode("utf-8"))  # assume that patches are also in utf-8
            fp.close()
            ps.items[0].source = tmpfile
            # 5. now do the patching
            success = ps.apply()
            if not success:
                raise errors.PatchError("Patch %s failed: %s" % (patchpath, pbuf.getvalue()))
            else:
                # 6. Finally get a patch description
                if ps.items[0].hunks[0].desc:
                    desc = ps.items[0].hunks[0].desc
                elif os.path.exists(descpath):
                    desc = util.readfile(descpath)
                else:
                    desc = "(No patch description available)"
                return util.readfile(tmpfile), desc
        else:
            return (text, None)

    def make_document(self, basefile=None):
        """
        Create a :py:class:`~ferenda.Document` objects with basic
        initialized fields.

        .. note::

           Helper method used by the
           :py:func:`~ferenda.decorators.makedocument` decorator.
        
        :param basefile: The basefile for the document
        :type  basefile: str
        :rtype: ferenda.Document
        """
        doc = Document()
        doc.basefile = basefile
        doc.meta = self.make_graph()
        doc.lang = self.lang
        doc.body = Body()
        if basefile:
            doc.basefile = basefile
            doc.uri = self.canonical_uri(basefile)
        return doc

    def make_graph(self):
        """
        Initialize a rdflib Graph object with proper namespace prefix
        bindings (as determined by
        :py:data:`~ferenda.DocumentRepository.namespaces`)

        :rtype: rdflib.Graph
        """
        g = Graph()
        for prefix, uri in list(self.ns.items()):
            # print "Binding %s to %s" % (prefix,uri)
            g.bind(prefix, uri)
        return g

    def create_external_resources(self, doc):
        """Optionally create external files that go together with the
        parsed file (stylesheets, images, etc).

        The default implementation does nothing.

        :param doc: The document
        :type  doc: ferenda.Document
        """

    def render_xhtml(self, doc, outfile=None):
        """Renders the parsed object structure as a XHTML file with
        RDFa attributes (also returns the same XHTML as a string).

        :param doc: The document to render
        :type  doc: ferenda.Document
        :param outfile: The file name for the XHTML document
        :type  outfile: str
        :returns: The XHTML document
        :rtype: str
        """
        xhtmldoc = self.render_xhtml_tree(doc)
        # Doctypes for XHTML+RDFa documents seem to be optional in RDFa 1.1
        # doctype = ('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML+RDFa 1.0//EN" '
        #           '"http://www.w3.org/MarkUp/DTD/xhtml-rdfa-1.dtd">')
        res = etree.tostring(xhtmldoc,
                             pretty_print=True,
                             xml_declaration=True,
                             encoding='utf-8',
                             # doctype=doctype
        )
        fileno, tmpfile = mkstemp()
        fp = os.fdopen(fileno)
        fp.close()
        with open(tmpfile, "wb") as fp:
            fp.write(res)
        util.replace_if_different(tmpfile, outfile)
        return res

    def render_xhtml_tree(self, doc):
        XML_LANG = "{http://www.w3.org/XML/1998/namespace}lang"
        XSI_SCHEMALOC = "{http://www.w3.org/2001/XMLSchema-instance}schemaLocation"
        def render_head(g, uri, children=None):
            if not children:
                children = []
                # if revlink == True, we're serializing triples for
                # the main subject. This also means we don't have to
                # set @about below, and that we should create a
                # <title> tag for any dct:title triple (ideally, for
                # any property that is rdfs:subPropertyOf dct:title,
                # but...
                revlink = True
            else:
                revlink = False
            # we sort to get a predictable order (by predicate, then by object)
            for (subj, pred, obj) in sorted(g, key=lambda t: (t[1], t[2])):
                if str(subj) != uri and str(obj) != uri:
                    # This isn't a triple we should serialize to RDFa,
                    # at least not in this iteration
                    continue

                if g.qname(pred) == "dct:title" and revlink:
                    if obj.language:
                        children.append(
                            E.title({'property': 'dct:title', }, str(obj)))
                    else:
                        children.append(E.title({'property':
                                        'dct:title', XML_LANG: ""}, str(obj)))

                elif isinstance(obj, URIRef) and str(subj) == uri:
                    children.append(E.link({'rel': g.qname(pred),
                                            'href': str(obj)}))
                    if not revlink:
                        children[-1].set('about', uri)
                    if str(obj) == doc.uri:
                        self.log.warning("Avoiding serializing circular graph (%s)" % doc.uri)
                    else:
                        render_head(g, str(obj), children)
                    
                elif isinstance(obj, URIRef):
                    if revlink:
                        children.append(E.link({'rev': g.qname(pred),
                                                'href': str(subj)}))
                elif obj.datatype:
                    children.append(E.meta({'property': g.qname(pred),
                                            'datatype': g.qname(obj.datatype),
                                            'content': str(obj)}))
                    if not revlink:
                        children[-1].set('about', uri)
                elif obj.language:
                    children.append(E.meta({'property': g.qname(pred),
                                            XML_LANG: obj.language,
                                            'content': str(obj)}))
                    if not revlink:
                        children[-1].set('about', uri)
                else:
                    children.append(E.meta({'property': g.qname(pred),
                                            'content': str(obj),
                                            XML_LANG: ''}))
                    if not revlink:
                        children[-1].set('about', uri)

            return E.head({'about': uri}, *children)

        nsmap = {None: "http://www.w3.org/1999/xhtml"}

        for prefix, namespace in self.ns.items():
            nsmap[prefix] = str(namespace)
        E = ElementMaker(namespace="http://www.w3.org/1999/xhtml",
                         nsmap=nsmap)
        headcontent = render_head(doc.meta, doc.uri)
        bodycontent = doc.body.as_xhtml(doc.uri)
        htmlattrs = {XSI_SCHEMALOC: "http://www.w3.org/1999/xhtml http://www.w3.org/MarkUp/SCHEMA/xhtml-rdfa-2.xsd",
                     "version": "XHTML+RDFa 1.1"}
        if doc.lang:
            htmlattrs[XML_LANG] = doc.lang
        xhtmldoc = E.html(
            htmlattrs,
            headcontent,
            bodycontent,
        )
        return xhtmldoc

    def parsed_url(self, basefile):
        """Get the full local url for the parsed file for the
        given basefile.
        
        :param basefile: The basefile for which to calculate the local url
        :type  basefile: str
        :returns: The local url
        :rtype: str
        """
        return self.generic_url(basefile, 'parsed', '.xhtml')

    def distilled_url(self, basefile):
        """Get the full local url for the distilled RDF/XML file for the
        given basefile.
        
        :param basefile: The basefile for which to calculate the local url
        :type  basefile: str
        :returns: The local url
        :rtype: str
        """
        return self.generic_url(basefile, 'distilled', '.rdf')

    #
    #
    # STEP 3: Extract and store the RDF data
    #
    #
    @classmethod
    def relate_all_setup(cls, config):
        """Runs any cleanup action needed prior to relating all documents in
        a docrepo. The default implementation clears the corresponsing
        context (see :py:meth:`~ferenda.DocumentRepository.dataset_uri`)
        in the triple store.

        .. note::

           Like :py:meth:`~ferenda.DocumentRepository.parse_all_setup`
           this might change to a instance method.

        Returns False if no relation needs to be done (as determined
        by the timestamp on the dump nt file)

        """
        # FIXME: should use dataset_uri(), but that's a instancemethod
        context = "%sdataset/%s" % (config.url, cls.alias)

        docstore = DocumentStore(config.datadir + os.sep + cls.alias)
        dump = docstore.path("dump", "distilled", ".nt")

        if not config.force:  # check if we need to work at all
            distilled = []
            for basefile in docstore.list_basefiles_for("relate"):
                distilled.append(docstore.distilled_path(basefile))
            if util.outfile_is_newer(distilled, dump):
                return False

        store = TripleStore.connect(config.storetype,
                                    config.storelocation,
                                    config.storerepository)
        log = cls._setup_logger(cls.alias)
        log.info("Clearing context %s at repository %s" % (
            context, config.storerepository))
        store.clear(context)

        # we can't clear the whoosh index in the same way as one index
        # contains documents from all repos. But we need to be able to
        # clear it from time to time, maybe with a clear/setup method
        # in manager? Or fulltextindex maybe could have a clear method
        # that removes all documents for a particular repo?
        return True

    @classmethod
    def relate_all_teardown(cls, config):
        """ 
        Runs any cleanup action needed after relating all documents in
        a docrepo. The default implementation does nothing.

        .. note::

           Like :py:meth:`~ferenda.DocumentRepository.parse_all_setup`
           this might change to a instance method.
        """
        # FIXME: should use dataset_uri(), but that's a instancemethod
        context = "%sdataset/%s" % (config.url, cls.alias)
        store = TripleStore.connect(config.storetype,
                                    config.storelocation,
                                    config.storerepository)
        docstore = DocumentStore(config.datadir + os.sep + cls.alias)
        dump = docstore.path("dump", "distilled", ".nt")
        log = cls._setup_logger(cls.alias)
        util.ensure_dir(dump)
        store.get_serialized_file(dump, format="nt", context=context)
        # just to report the number of dumped triples -- may be unneccesary
        log.info("Dumped %(triplecount)s triples from context %(context)s to %(dumpfile)s" %
                 {'triplecount': sum(1 for line in open(dump)),
                  'context': context,
                  'repository': config.storerepository,
                  'dumpfile': dump})
        return True

    def relate(self, basefile, otherrepos=[]):
        """Runs various indexing operations for the document represented by
           *basefile*: insert RDF statements into a triple store, add this
           document to the dependency list to all documents that it refers to,
           and put the text of the document into a fulltext index.

        """
        self.relate_triples(basefile)

        # When otherrepos = [], should we still provide self as one repo? Yes.
        if self not in otherrepos:
            otherrepos.append(self)

        self.relate_dependencies(basefile, otherrepos)
        if self.config.fulltextindex:
            self.relate_fulltext(basefile)

    def _get_triplestore(self, **kwargs):
        if not hasattr(self, '_triplestore'):
            self._triplestore = TripleStore.connect(self.config.storetype,
                                                    self.config.storelocation,
                                                    self.config.storerepository,
                                                    **kwargs)
        return self._triplestore

    def relate_triples(self, basefile):
        """Insert the (previously distilled) RDF statements into the
        triple store.

        :param basefile: The basefile for the document containing the
                         RDF statements.
        :type  basefile: str                        
        :returns: None
        """
        ts = self._get_triplestore()  # init self._triplestore
        with util.logtime(self.log.debug,
                          "%(basefile)s: Added %(rdffile)s to context %(context)s in %(elapsed).3f sec",
                          {'basefile': basefile,
                           'context': self.dataset_uri(),
                           'dataset': self.dataset_uri(),
                           'rdffile': self.store.distilled_path(basefile),
                           'triplestore': self.config.storelocation}):
            data = open(self.store.distilled_path(basefile), "rb").read()
            ts.add_serialized(data, format="xml", context=self.dataset_uri())

    def _get_fulltext_indexer(self, batchoptimize=False):
        if not hasattr(self, '_fulltextindexer'):

            idx = FulltextIndex.connect(self.config.indextype,
                                        self.config.indexlocation,
                                        repos=[])  # FIXME: need a real list of repos
            self._fulltextindexer = idx

            # The batchwriter functionality seems a litte broken --
            # gave a "ValueError: seek of closed file" error. Since
            # it's used to speed things up, and we now have
            # ElasticSearch support for that, it's disabled until
            # further notice.
            # if hasattr(self.config, 'all'):
            #     self._fulltextindexer._batchwriter = True

        return self._fulltextindexer

    def relate_dependencies(self, basefile, repos=[]):
        """For each document that the basefile document refers to, attempt to
find this document in the current or any other docrepo, and add the
parsed document path to that documents dependency file."""
        values = {'basefile': basefile,
                  'deps': 0}
        with util.logtime(self.log.debug,
                          "%(basefile)s: Registered %(deps)s dependencies in %(elapsed).3f sec",
                          values):

            with self.store.open_distilled(basefile) as fp:
                g = Graph().parse(fp, format="xml")
                for (s, p, o) in g:
                    # for each URIRef in graph
                    if isinstance(o, URIRef):
                        for repo in repos:
                            # find out if any docrepo can handle it
                            dep_basefile = repo.basefile_from_uri(str(o))
                            if dep_basefile and dep_basefile != basefile:
                                # self.log.debug("basefile %s adds dependency to %s" % (basefile, dep_basefile))
                                # if so, add to that repo's dependencyfile
                                repo.add_dependency(dep_basefile,
                                                    self.store.parsed_path(basefile))
                                values['deps'] += 1
        return values['deps']

    def add_dependency(self, basefile, dependencyfile):
        """Add the *dependencyfile* to *basefile* s dependency file. Returns
        True if anything new was added, False otherwise

        """
        present = False
        if os.path.exists(self.store.dependencies_path(basefile)):
            with self.store.open_dependencies(basefile) as fp:
                for line in fp:
                    if isinstance(line, bytes):
                       line = line.decode('utf-8') 
                    if line.strip() == dependencyfile:
                        present = True
        if not present:
            with self.store.open_dependencies(basefile, "ab") as fp:
                fp.write((dependencyfile+os.linesep).encode("utf-8"))


        return not present  # return True if we added something, False otherwise

    def relate_fulltext(self, basefile):
        """Index the text of the document into fulltext index.
        
        :param basefile: The basefile for the document to be indexed.
        :type  basefile: str                        
        :returns: None
        """
        values = {'basefile': basefile,
                  'resources': 0,
                  'words': 0}
        with util.logtime(self.log.debug,
                          "%(basefile)s: Added %(resources)s resources (%(words)s words) to fulltext index in %(elapsed).3f s", values):
            indexer = self._get_fulltext_indexer()
            tree = etree.parse(self.store.parsed_path(basefile))
            g = Graph()
            desc = Describer(g.parse(data=util.readfile(self.store.distilled_path(basefile))))
            dct = self.ns['dct']

            body = tree.find(".//{http://www.w3.org/1999/xhtml}body")
            for resource in [body] + body.findall(".//*[@about]"):
                if resource.tag == "{http://www.w3.org/1999/xhtml}head":
                    continue
                about = resource.get('about')
                if isinstance(about, bytes):  # happens under py2
                    about = about.decode()    # pragma: no cover
                desc.about(about)
                repo = self.alias
                if isinstance(repo, bytes):  # again, py2
                    repo = repo.decode()     # pragma: no cover
                plaintext = self._extract_plaintext(resource)
                l = desc.getvalues(dct.title)
                title = str(l[0]) if l else None
                l = desc.getvalues(dct.identifier)
                identifier = str(l[0]) if l else None
                indexer.update(uri=about,
                               repo=repo,
                               basefile=basefile,
                               title=title,
                               identifier=identifier,
                               text=plaintext)
                values['resources'] += 1
                values['words'] += len(plaintext.split())

            indexer.commit()  # NB: Destroys indexer._writer

    def _extract_plaintext(self, node):
        # helper to extract any text from a elementtree node,
        # excluding subnodes that are resources themselves (ie they
        # have an @about node)
        plaintext = node.text if node.text else ""
        for subnode in node:
            if not subnode.get('about'):
                plaintext += self._extract_plaintext(subnode)
        if node.tail:
            plaintext += node.tail
        # append trailing space for block-level elements (including
        # <br>, <img> and some others that formally are inline
        # elements)
        trailspace = "" if node.tag in ("a" "b", "i", "span") else " "
        return plaintext.strip() + trailspace
    #
    #
    # STEP 4: Generate browser-ready HTML with navigation panels,
    # information about related documents and so on.
    #
    #

    @classmethod
    def generate_all_setup(cls, config):
        """ 
        Runs any action needed prior to generating all documents in a
        docrepo. The default implementation does nothing.

        .. note::

           Like :py:meth:`~ferenda.DocumentRepository.parse_all_setup`
           this might change to a instance method.
        """

    @classmethod
    def generate_all_teardown(cls, config):
        """ 
        Runs any cleanup action needed after generating all documents
        in a docrepo. The default implementation does nothing.

        .. note::

           Like :py:meth:`~ferenda.DocumentRepository.parse_all_setup`
           this might change to a instance method.
        """

    @decorators.action
    def generate(self, basefile, otherrepos=[]):
        """Generate a browser-ready HTML file from structured XML and RDF.

        Uses the XML and RDF files constructed by
        :py:meth:`ferenda.DocumentRepository.parse`.

        The generation is done by XSLT, and normally you won't need to
        override this, but you might want to provide your own xslt
        file and set
        :py:data:`ferenda.DocumentRepository.xslt_template` to the
        name of that file.

        If you want to generate your browser-ready HTML by any other
        means than XSLT, you should override this method.

        :param basefile: The basefile for which to generate HTML
        :type  basefile: str
        :returns: None
        """
        with util.logtime(self.log.info, "%(basefile)s: OK (%(elapsed).3f sec)",
                          {'basefile': basefile}):
            # This dependency management could be abstracted away like
            # the parseifneeded decorator does for parse(). But unlike
            # parse(), noone is expected to override generate(), so
            # the proper place to handle this complexity is probably
            # here.
            infile = self.store.parsed_path(basefile)
            annotations = self.store.annotation_path(basefile)
            if os.path.exists(self.store.dependencies_path(basefile)):
                deptxt = util.readfile(self.store.dependencies_path(basefile))
                dependencies = deptxt.strip().split("\n")
            else:
                dependencies = []
            dependencies.extend((infile, annotations))

            outfile = self.store.generated_path(basefile)
            if ((not self.config.force) and
                    util.outfile_is_newer(dependencies, outfile)):
                self.log.debug("%s: Skipped", basefile)
                return
            self.log.debug("%s: Starting", basefile)

            # All bookkeping done, now lets prepare and transform!

            # The annotationfile might be newer than all dependencies
            # (and thus not need regenerateion) even though the
            # outfile is older.
            if (self.config.force or (not
                    util.outfile_is_newer(dependencies, self.store.annotation_path(basefile)))):
                with util.logtime(self.log.debug,
                                  "%(basefile)s: prep_annotation_file in %(elapsed).3f sec",
                                  {'basefile': basefile}):
                    # annotation_file should be the same as annotations above?
                    annotation_file = self.prep_annotation_file(basefile)
            else:
                annotation_file = self.store.annotation_path(basefile)
            params = {}
            if annotation_file:
                params['annotationfile'] = annotation_file

            with util.logtime(self.log.debug,
                              "%(basefile)s: transform in %(elapsed).3f",
                              {'basefile': basefile}):
                conffile = os.path.abspath(
                    os.sep.join([self.config.datadir, 'rsrc', 'resources.xml']))
                transformer = Transformer('XSLT', self.xslt_template,
                                          ["res/xsl"], config=conffile,
                                          documentroot=self.config.datadir)
                urltransform = None
                if self.config.staticsite:
                    repos = list(otherrepos)
                    if self not in repos:
                        repos.append(self)
                    urltransform = self.get_url_transform_func(repos, os.path.dirname(outfile))

                transformer.transform_file(infile, outfile,
                                           params, urltransform)

            # At this point, outfile may appear untouched if it already
            # existed and wasn't actually changed. But this will cause the
            # above outfile_is_newer check to fail next time around. Also,
            # the docentry.updated parameter will be incosistent with the
            # timestamp on the file. What to do?
            os.utime(outfile, None)  # update access/modified timestamp
            now = datetime.now()
            docentry = DocumentEntry(self.store.documententry_path(basefile))
            if not docentry.published:
                docentry.published = now
            docentry.updated = now
            docentry.save()

    def get_url_transform_func(self, repos, basedir):
        # This implementation always transforms URLs to local file
        # paths (or if they can't be mapped, leaves them alone)

        # FIXME: apply some memoization to this
        def transform(uri):
            path = None
            if uri == self.config.url:
                path = self.config.datadir + os.sep + "index.html"
                # path = basedir + os.sep + "index.html"
            else:
                for repo in repos:
                    basefile = repo.basefile_from_uri(uri)
                    # 2-tuple, empty tuple, or none
                    dataset_params = repo.dataset_params_from_uri(uri)
                    if basefile or (dataset_params is not None):
                        break
                if basefile:
                    path = repo.store.generated_path(basefile)
                elif dataset_params is not None:
                    # FIXME: This reimplements the logic that calculates
                    # basefile at the end of toc_pagesets
                    if dataset_params:
                        pseudobasefile = "/".join(dataset_params)
                    else:
                        pseudobasefile = "index"
                    path = repo.store.path(pseudobasefile, 'toc', '.html')
            if path:
                relpath = os.path.relpath(path, basedir)
                if os.sep == "\\":
                    relpath = relpath.replace(os.sep, "/")
                return relpath

            else:
                return uri
        return transform

    def prep_annotation_file(self, basefile):
        """Helper function used by
        :py:meth:`~ferenda.DocumentRepository.generate` -- prepares a
        RDF/XML file containing statements that in some way annotates
        the information found in the document that generate handles,
        like URI/title of other documents that refers to this one.

        :param basefile: The basefile for which to collect annotating
                         statements.
        :type basefile: str
        :returns: The full path to the prepared RDF/XML file
        :rtype: str

        """
        # return self.store.annotation_path(basefile)
        graph = self.construct_annotations(self.canonical_uri(basefile))
        if graph and len(graph) > 0:
            with self.store.open_annotation(basefile, "w") as fp:
                fp.write(self.graph_to_annotation_file(graph))
            return self.store.annotation_path(basefile)
        else:
            self.log.warning(
                "%s: No annotation data fetched, something might be wrong with the SPARQL query" % basefile)

    def construct_annotations(self, uri):
        """Construct a RDF graph containing metadata relating to *uri* in some
        way, using the query template specified by
        :data:`~ferenda.DocumentRepository.sparql_annotations`

        """
        sq = self.construct_sparql_query(uri)
        if self.config.storelocation:
            kwargs = {}
            if self.config.storetype in ("SQLITE", "SLEEPYCAT"):
                kwargs['inmemory'] = True
            ts = self._get_triplestore(**kwargs)
            res = ts.construct(sq)
            # bind namespaces so that the constructed graph looks pretty
            for prefix, uri in list(self.ns.items()):
                res.bind(prefix, uri)

            return res

    def construct_sparql_query(self, uri):
        query_template = self.sparql_annotations
        if os.path.exists(query_template):
            fp = open(query_template, 'rb')
        elif pkg_resources.resource_exists('ferenda', query_template):
            fp = pkg_resources.resource_stream('ferenda', query_template)
        else:
            raise ValueError("query template %s not found" % query_template)
        params = {'uri': uri}
        sq = fp.read().decode('utf-8') % params
        fp.close()
        return sq
        

    # helper for the prep_annotation_file helper -- it expects a
    # RDFLib graph, and returns a XML string in Grit format
    def graph_to_annotation_file(self, graph):
        """Converts a RDFLib graph into a XML file with the same
        statements, ordered using the Grit format
        (https://code.google.com/p/oort/wiki/Grit) for easier XSLT
        inclusion.

        :param graph: The graph to convert
        :type  graph: rdflib.graph.Graph
        :returns: A serialized XML document with the RDF statements 
        :rtype: str
        """
        fp = BytesIO(graph.serialize(format="xml"))
        intree = etree.parse(fp)
        fp = pkg_resources.resource_stream('ferenda', "res/xsl/rdfxml-grit.xsl")
        transform = etree.XSLT(etree.parse(fp))
        resulttree = transform(intree)
        res = etree.tostring(resulttree, pretty_print=format)
        return res.decode('utf-8')

    # the inverse of graph_to_annotation_file
    def annotation_file_to_graph(self, annotation_file):
        """Converts a annotation file (using the Grit format) back into an RDFLib graph.

        :param graph: The filename of a serialized XML document with RDF statements
        :type  graph: str
        :returns: The RDF statements as a regular graph
        :rtype: rdflib.Graph
        """
        with open(annotation_file, "rb") as fp:
            intree = etree.parse(fp)
        fp = pkg_resources.resource_stream('ferenda', "res/xsl/grit-grddl.xsl")
        transform = etree.XSLT(etree.parse(fp))
        resulttree = transform(intree)
        res = etree.tostring(resulttree, pretty_print=format)
        g = Graph()
        g.parse(data=res)
        return g

    def generated_url(self, basefile):
        """Get the full local url for the generated file for the
        given basefile.
        
        :param basefile: The basefile for which to calculate the local url
        :type  basefile: str
        :returns: The local url
        :rtype: str
        """
        return self.generic_url(basefile, 'generated', '.html')

    #
    #
    # STEP 5: Generate HTML pages for a TOC of a all documents, news
    # pages of new/updated documents, and other odds'n ends.
    #
    #

    # toc
    #     toc_select
    #         toc_query
    #              (toc_predicates ?)
    #     toc_criteria
    #         toc_predicates
    #         toc_selector
    #     toc_pagesets
    #         (selectors)
    #     toc_select_for_pages   <-- where most of the the magic happens
    #         (selectors)
    #         toc_item
    #     toc_generate_pages
    #         toc_generate_page
    #
    def toc(self, otherrepos=[]):
        """Creates a set of pages that together acts as a table of
        contents for all documents in the repository. For smaller
        repositories a single page might be enough, but for
        repositoriees with a few hundred documents or more, there will
        usually be one page for all documents starting with A,
        starting with B, and so on. There might be different ways of
        browseing/drilling down, i.e. both by title, publication year,
        keyword and so on.

        The default implementation calls
        :py:meth:`~ferenda.DocumentRepository.toc_select` to get all
        data from the triple store,
        :py:meth:`~ferenda.DocumentRepository.toc_criteria` to find
        out the criteria for ordering,
        :py:meth:`~ferenda.DocumentRepository.toc_pagesets` to
        calculate the total set of TOC html files,
        :py:meth:`~ferenda.DocumentRepository.toc_select_for_pages` to
        create a list of documents for each TOC html file, and finally
        :py:meth:`~ferenda.DocumentRepository.toc_generate_pages` to
        create the HTML files. The default implemention assumes that
        documents have a title (in the form of a ``dct:title``
        property) and a publication date (in the form of a
        ``dct:issued`` property). 

        You can override any of these methods to customize any part of
        the toc generation process. Often overriding :py:meth:`~ferenda.DocumentRepository.toc_criteria` to
        specify other document properties will be sufficient."""

        params = {}
        with util.logtime(self.log.debug,
                          "toc: selected %(rowcount)s rows in %(elapsed).3f s",
                          params):
            data = self.toc_select(self.dataset_uri())
            params['rowcount'] = len(data)
        if len(data) > 0:
            criteria = self.toc_criteria(self.toc_predicates())
            pagesets = self.toc_pagesets(data, criteria)
            pagecontent = self.toc_select_for_pages(data, pagesets, criteria)
            self.toc_generate_pages(pagecontent, pagesets, otherrepos)
            self.toc_generate_first_page(pagecontent, pagesets, otherrepos)
        else:
            self.log.error("toc_select found 0 results for query, can't generate TOC")
            self.log.info("(query PROBABLY was '%s')" % self.toc_query())

    def toc_select(self, context=None):
        """Select all data from the triple store needed to make up all
        TOC pages.

        :param context: The context (named graph) to restrict the query to.
                        If None, search entire triplestore.
        :type  context: str
        :returns: The results of the query, as python objects
        :rtype: set of dicts"""

        store = TripleStore.connect(self.config.storetype,
                                    self.config.storelocation,
                                    self.config.storerepository)

        sq = self.toc_query(context)
        res = store.select(sq, "python")
        store.close()
        return res

    def toc_query(self, context=None):
        """Constructs a SPARQL SELECT query that fetches all
        information needed to construct the complete set of TOC pages
        in the form of a single list of result rows.

        Override this method if you need to customize the query.

        :param context: The context (named graph) to which to limit
                        the query. If None, query the entire
                        triplestore.
        :type  context: str
        :returns: The SPARQL query
        :rtype: str

        Example:

        >>> d = DocumentRepository()
        >>> expected = 'PREFIX bibo: <http://purl.org/ontology/bibo/> PREFIX dct: <http://purl.org/dc/terms/> PREFIX foaf: <http://xmlns.com/foaf/0.1/> PREFIX owl: <http://www.w3.org/2002/07/owl#> PREFIX prov: <http://www.w3.org/ns/prov-o/> PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> PREFIX skos: <http://www.w3.org/2004/02/skos/core#> PREFIX xhv: <http://www.w3.org/1999/xhtml/vocab#> PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> PREFIX xsi: <http://www.w3.org/2001/XMLSchema-instance> SELECT DISTINCT ?uri ?title ?issued FROM <http://example.org/ctx/base> WHERE {?uri rdf:type foaf:Document ; dct:title ?title . OPTIONAL { ?uri dct:issued ?issued . }  }'
        >>> d.toc_query("http://example.org/ctx/base") == expected
        True
        """
        from_graph = ""
        if context:
            from_graph = "FROM <%s>" % context
        elif self.config.storetype == "FUSEKI":
            from_graph = "FROM <urn:x-arq:UnionGraph>"

        predicates = self.toc_predicates()
        g = self.make_graph()
        bindings = " ".join(["?" + util.uri_leaf(b) for b in predicates])
        whereclause = "?uri rdf:type %s ; %s ?%s" % (g.qname(self.rdf_type),
                                                     g.qname(predicates[0]),
                                                     util.uri_leaf(predicates[0]))
        optclauses = "".join(
            ["OPTIONAL { ?uri %s ?%s . } " % (g.qname(b), util.uri_leaf(b)) for b in predicates[1:]])

        # FIXME: The above doctest looks like crap since all
        # registered namespaces in the repo is included. Should only
        # include prefixes actually used
        prefixes = " ".join(["PREFIX %s: <%s>" % (p, u) for p, u in sorted(self.ns.items())])
        query = "%s SELECT DISTINCT ?uri %s %s WHERE {%s . %s }" % (
            prefixes, bindings, from_graph, whereclause, optclauses)
        return query

    def toc_criteria(self, predicates=None):
        """Create the criteria used to organize the documents in the
        repository into different pagesets.

        :param predicates: The :py:class:`~rdflib.term.URIRef` terms to use as base for criteria
        :type  predicates: list
        :returns: :py:class:`~ferenda.TocCriteria`
                  objects, each representing a particular way of organizing the
                  documents, and each corresponding to a TocPageset object (constructed
                  by :py:meth:`~ferenda.DocumentRepository.toc_pagesets`)
        :rtype: list
        """

        criteria = []
        for predicate in predicates:
            # make an appropriate selector etc. a proper implementation
            # would look at the ontology of the predicate, take a look
            # at the range for that DataProperty and select an
            # appropriate selector.
            if predicate == self.ns['dct']['issued']:  # date property
                selector = lambda x: x['issued'][:4]
                key = selector
                label = 'Sorted by publication year'
                pagetitle = 'Documents published in %(select)s'
                selector_descending = True

            else:
                # selector and key for proper title sort
                # (eg. disregarding leading "the", not counting
                # spaces)
                def sortkey(d):
                    title = d['title'].lower()
                    if title.startswith("the "):
                        title = title[4:]
                    # filter away starting non-word characters (but not digits)
                    title = re.sub("^\W+", "", title)
                    # remove spaces
                    return "".join(title.split())

                selector = lambda x: sortkey(x)[0]
                key = sortkey
                label = label = 'Sorted by ' + util.uri_leaf(predicate)
                pagetitle = 'Documents starting with "%(select)s"'
                selector_descending = False

            criteria.append(TocCriteria(binding=util.uri_leaf(predicate).lower(),
                                        label=label,
                                        pagetitle=pagetitle,
                                        selector=selector,
                                        key=key,
                                        selector_descending=selector_descending))
        return criteria

    def toc_predicates(self):
        """Return a list of predicates (as
        :py:class:`~rdflib.term.URIRef` objects that each should be
        used to organize a table of contents of documents in this
        docrepo).

        Is used by toc_criteria, must match results from sparql query
        in toc_query."""

        return [self.ns['dct'].title,
                self.ns['dct'].issued]

    def toc_pagesets(self, data, criteria):
        """Calculate the set of needed TOC pages based on the result rows

        :param data: list of dicts, each dict containing metadata about a single document
        :param criteria: list of TocCriteria objects
        :returns: The link text, page title and base file for each needed
                  TOC page, structured by selection criteria.
        :rtype: 3-dimensional named tuple

        Example:
        
        >>> d = DocumentRepository()
        >>> rows = [{'uri':'http://ex.org/1','title':'Abc','issued':'2009-04-02'},
        ...         {'uri':'http://ex.org/2','title':'Abcd','issued':'2010-06-30'},
        ...         {'uri':'http://ex.org/3','title':'Dfg','issued':'2010-08-01'}]
        >>> from operator import itemgetter
        >>> criteria = (TocCriteria(binding='title',
        ...                         label='By title',
        ...                         pagetitle='Documents starting with "%(select)s"',
        ...                         selector=lambda x: x['title'][0].lower(),
        ...                         key=itemgetter('title')),
        ...             TocCriteria(binding='issued',
        ...                         label='By publication year',
        ...                         pagetitle='Documents published in %(select)s',
        ...                         selector=lambda x: x['issued'][:4],
        ...                         key=itemgetter('issued')))
        >>> # Note: you can get a suitable tuple of TocCriteria
        >>> # objects by calling toc_criteria() as well
        >>> pagesets=d.toc_pagesets(rows,criteria)
        >>> pagesets[0].label == 'By title'
        True
        >>> pagesets[0].pages[0] == TocPage(linktext='a', title='Documents starting with "a"', binding='title', value='a')
        True
        >>> pagesets[0].pages[0].linktext == 'a'
        True
        >>> pagesets[0].pages[0].title == 'Documents starting with "a"'
        True
        >>> pagesets[0].pages[0].binding == 'title'
        True
        >>> pagesets[0].pages[0].value == 'a'
        True
        >>> pagesets[1].label == 'By publication year'
        True
        >>> pagesets[1].pages[0] == TocPage(linktext='2009', title='Documents published in 2009', binding='issued', value='2009')
        True
        """

        res = []
        for criterion in criteria:
            pageset = TocPageset(label=criterion.label, pages=[])
            selector_values = {}
            selector = criterion.selector
            binding = criterion.binding
            for row in data:
                try:
                    selector_values[selector(row)] = True
                except KeyError as e:
                    # this will happen a lot on simple selector
                    # functions when handed incomplete data
                    pass

            for value in sorted(list(selector_values.keys()), reverse=criterion.selector_descending):
                pageset.pages.append(TocPage(linktext=value,
                                             title=criterion.pagetitle % {'select': value},
                                             binding=binding,
                                             value=value))
            res.append(pageset)
        return res

    def toc_select_for_pages(self, data, pagesets, criteria):
        """Go through all data rows (each row representing a document)
        and, for each toc page, select those documents that are to
        appear in a particular page.
        
        Example:
        
        >>> d = DocumentRepository()
        >>> rows = [{'uri':'http://ex.org/1','title':'Abc','issued':'2009-04-02'},
        ...         {'uri':'http://ex.org/2','title':'Abcd','issued':'2010-06-30'},
        ...         {'uri':'http://ex.org/3','title':'Dfg','issued':'2010-08-01'}]
        >>> from rdflib import Namespace
        >>> dct = Namespace("http://purl.org/dc/terms/")
        >>> criteria = d.toc_criteria([dct.title,dct.issued])
        >>> pagesets=d.toc_pagesets(rows,criteria)
        >>> expected={('title','a'):[[Link('Abc',uri='http://ex.org/1')],
        ...                          [Link('Abcd',uri='http://ex.org/2')]],
        ...           ('title','d'):[[Link('Dfg',uri='http://ex.org/3')]],
        ...           ('issued','2009'):[[Link('Abc',uri='http://ex.org/1')]],
        ...           ('issued','2010'):[[Link('Abcd',uri='http://ex.org/2')],
        ...                              [Link('Dfg',uri='http://ex.org/3')]]}
        >>> d.toc_select_for_pages(rows, pagesets, criteria) == expected
        True
         
        :param data: List of dicts as returned by :meth:`~ferenda.DocumentRepository.toc_select`
        :param pagesets: Result from :meth:`~ferenda.DocumentRepository.toc_pagesets`
        :param criteria: Result from :meth:`~ferenda.DocumentRepository.toc_criteria`
        :returns: mapping between toc basefile and documentlist for that basefile
        :rtype: dict
        """

        # to 1-dimensional dict (odict?): {(binding,value): [list-of-Elements]}
        res = {}
        for pageset, criterion in zip(pagesets, criteria):
            documents = defaultdict(list)
            for row in data:
                try:
                    key = criterion.selector(row)
                    documents[key].append(row)
                except KeyError:
                    pass
            for key in documents.keys():
                # find appropriate page in pageset and read it's basefile
                for page in pageset.pages:
                    if page.linktext == key:
                        s = sorted(documents[key],
                                   key=criterion.key,
                                   reverse=criterion.key_descending)
                        res[(page.binding, page.value)] = [self.toc_item(criterion.binding, row)
                                                           for row in s]
        return res

    def toc_item(self, binding, row):
        """Returns a formatted version of row, using Element objects"""
        # default impl always just a simple link with title as link text
        return [Link(row['title'],  # yes, ignore binding
                     uri=row['uri'])]

    # pagecontent -> documentlists?
    def toc_generate_pages(self, pagecontent, pagesets, otherrepos=[]):
        """Creates a set of TOC pages by calling
         :meth:`~ferenda.DocumentRepository.toc_generate_page`.

        :param pagecontent: Result from
                            :meth:`~ferenda.DocumentRepository.toc_select_for_pages`
        :param pagesets: Result from
                         :meth:`~ferenda.DocumentRepository.toc_pagesets`
        :param otherrepos: A list of document repository instances

        """
        paths = []
        for (binding, value), documents in sorted(pagecontent.items()):
            paths.append(self.toc_generate_page(
                binding, value, documents, pagesets, None, otherrepos))
        return paths

    def toc_generate_first_page(self, pagecontent, pagesets, otherrepos=[]):
        """Generate the main page of TOC pages."""
        firstpage = pagesets[0].pages[0]  # has .binding and .value
        documents = pagecontent[(firstpage.binding, firstpage.value)]
        # (binding,value), documents = sorted(pagecontent.items(), key=itemgetter(0))[0]
        return self.toc_generate_page(firstpage.binding, firstpage.value, documents, pagesets, "index", otherrepos)

    def toc_generate_page(self, binding, value, documentlist, pagesets, effective_basefile=None, otherrepos=[]):
        """Generate a single TOC page.

        :param binding: The binding used (eg. 'title' or 'issued')
        :param value: The value for the used binding (eg. 'a' or '2013'
        :param documentlist: Result from :meth:`~ferenda.DocumentRepository.toc_select_for_pages`
        :param pagesets: Result from :meth:`~ferenda.DocumentRepository.toc_pagesets`
        :param effective_basefile: Place the resulting page somewhere else than ``toc/*binding*/*value*.html``
        :param otherrepos: A list of document repository instances
        """
        if effective_basefile == None:
            effective_basefile = binding + "/" + value
        outfile = self.store.path(effective_basefile, 'toc', '.html')
        doc = self.make_document()
        doc.uri = self.dataset_uri(binding, value)
        d = Describer(doc.meta, doc.uri)
        nav = UnorderedList(role='navigation')
        for pageset in pagesets:
            sublist = UnorderedList()
            for page in pageset.pages:
                if page.binding == binding and page.value == value:
                    title = page.title
                    sublist.append(ListItem([page.linktext]))
                else:
                    href = self.dataset_uri(page.binding, page.value)
                    sublist.append(ListItem([Link(str(page.linktext), href=href)]))
            nav.append(ListItem([Paragraph(pageset.label), sublist]))

        d.value(self.ns['dct'].title, title)

        # Consider other strategies; definition lists with
        # subheadings, orderedlists, tables...
        ul = UnorderedList([ListItem(x) for x in documentlist], role='main')
        doc.body = Body([nav,
                         ul
                         ])

        conffile = os.path.abspath(
            os.sep.join([self.config.datadir, 'rsrc', 'resources.xml']))
        transformer = Transformer('XSLT', "res/xsl/toc.xsl", ["res/xsl"],
                                  config=conffile)
        # FIXME: This is a naive way of calculating the relative depth
        # of the outfile
        depth = len(outfile[len(self.store.datadir) + 1:].split(os.sep))
        repos = [self] + otherrepos
        if self.config.staticsite:
            uritransform = self.get_url_transform_func(repos, os.path.dirname(outfile))
        else:
            uritransform = None
        tree = transformer.transform(self.render_xhtml_tree(doc), depth, uritransform=uritransform)

        fixed = transformer.t.html5_doctype_workaround(etree.tostring(tree))

        with self.store.open(effective_basefile, 'toc', '.html', "wb") as fp:
            fp.write(fixed)

        self.log.info("Created %s" % outfile)
        return outfile

    def news(self, otherrepos=[]):
        """Create a set of Atom feeds and corresponding HTML pages for
        new/updated documents in different categories in the
        repository.

        """

        criteria = self.news_criteria()
        for entry in self.news_entries():
            for criterion in criteria:
                if criterion.selector(entry):
                    criterion.entries.append(entry)
        conffile = os.path.abspath(
            os.sep.join([self.config.datadir, 'rsrc', 'resources.xml']))
        docroot = os.path.dirname(self.store.path('foo','feed','.x'))
        transformer = Transformer("XSLT", "res/xsl/atom.xsl", ["res/xsl"],
                                  docroot, config=conffile)
        for criterion in criteria:
            # should reverse=True be configurable? For datetime
            # properties it makes sense to use most recent first, but
            # maybe other cases?
            entries = sorted(criterion.entries, key=criterion.key, reverse=True)
            self.log.info("feed %s: %s entries" % (criterion.basefile, len(entries)))
            self.news_write_atom(entries,
                                 criterion.feedtitle,
                                 criterion.basefile)

            outfile = self.store.path(criterion.basefile, 'feed', '.html')
            transformer.transform_file(self.store.atom_path(criterion.basefile),
                                       outfile)

    def news_criteria(self):
        """Returns a list of NewsCriteria objects."""
        return [NewsCriteria('main', 'New and updated documents')]

    def news_entries(self):
        """Return a generator of all available entries, represented as tuples
        of (DocumentEntry, rdflib.Graph) objects. The Graph contains
        all distilled metadata about the document.

        """
        directory = os.path.sep.join((self.config.datadir, self.alias, "entries"))
        for basefile in self.store.list_basefiles_for("news"):
            path = self.store.documententry_path(basefile)
            entry = DocumentEntry(path)
            if not entry.published:
                # not published -> shouldn't be in feed
                continue
            if not entry.id:
                entry.id = self.canonical_uri(basefile)
            if not entry.url:
                entry.url = self.generated_url(basefile)
            if not entry.basefile:
                entry.basefile = basefile

            if not os.path.exists(self.store.distilled_path(basefile)):
                self.log.warn("%s: No distilled file at %s, skipping" % (basefile,
                                                                         self.store.distilled_path(basefile)))
                continue

            dct = self.ns['dct']
            if not entry.title:
                try:
                    g = Graph()
                    g.parse(self.store.distilled_path(basefile))
                    desc = Describer(g, entry.id)
                    entry.title = desc.getvalue(dct.title)
                except KeyError:  # no dct:title -- not so good
                    self.log.warn("%s: No title available" % basefile)
                    entry.title = entry.id

            try:
                entry.summary = desc.getvalue(dct.abstract)
            except KeyError:  # no dct:abstract -- that's OK
                pass

            # 4: Set links to RDF metadata and document content
            if not entry.link:
                entry.set_link(self.store.distilled_path(basefile),
                               self.distilled_url(basefile))

            # If we just republish eg. the original PDF file and don't
            # attempt to parse/enrich the document
            if not entry.content:
                if (self.config.republishsource):
                    entry.set_content(self.store.downloaded_path(basefile),
                                      self.downloaded_url(basefile))
                else:
                    # the parsed (machine reprocessable) version. The
                    # browser-ready version is referenced with the <link>
                    # element, separate from the set_link <link>
                    entry.set_content(self.store.parsed_path(basefile),
                                      self.parsed_url(basefile))
            yield entry

    def news_write_atom(self, entries, title, basefile, archivesize=1000):
        """Given a list of Atom entry-like objects, including links to RDF
        and PDF files (if applicable), create a rinfo-compatible Atom feed,
        optionally splitting into archives."""

        # This nested func does most of heavy lifting, the main
        # function code only sets up basic constants and splits the
        # entries list into appropriate chunks
        def write_file(entries, suffix="", prevarchive=None, nextarchive=None):
            feedfile = self.store.path(basefile + suffix, 'feed', '.atom')
            nsmap = {None: 'http://www.w3.org/2005/Atom',
                     'le': 'http://purl.org/atompub/link-extensions/1.0'}
            E = ElementMaker(nsmap=nsmap)

            # entries SHOULD at this point be a list of DocumentEntry
            # object, not (DocumentEntry, Graph).
            if entries:
                updated = max(entries, key=lambda x: x.updated).updated
            else:
                updated = datetime.now()  # or never
            contents = [E.id(feedid),
                        E.title(title),
                        E.updated(util.rfc_3339_timestamp(updated)),
                        E.author(
                            E.name("Ferenda"),
                            E.email("info@example.org"),
                            E.uri(self.config.url)
                        ),
                        E.link({'rel': 'self', 'href': feedurl})]
            if prevarchive:
                contents.append(E.link({'rel': 'prev-archive',
                                        'href': prevarchive}))
            if nextarchive:
                contents.append(E.link({'rel': 'next-archive',
                                        'href': nextarchive}))

            for entry in entries:
                entrynodes = [E.title(entry.title),
                              E.summary(str(entry.summary)),
                              E.id(entry.id),
                              E.published(util.rfc_3339_timestamp(entry.published)),
                              E.updated(util.rfc_3339_timestamp(entry.updated)),
                              E.link({'href': util.relurl(entry.url, feedurl)})]
                if entry.link:
                    node = E.link({'rel': 'alternate',
                                   'href': util.relurl(entry.link['href'],
                                                       feedurl),
                                   'type': entry.link['type'],
                                   'length': str(entry.link['length']),
                                   'hash': entry.link['hash']})
                    entrynodes.append(node)
                if entry.content and entry.content['markup']:
                    node = E.content({'type': 'xhtml'},
                                     etree.XML(entry.content['markup']))
                    entrynodes.append(node)
                elif entry.content and entry.content['src']:
                    node = E.content({'src': util.relurl(entry.content['src'],
                                                         feedurl),
                                      'type': entry.content['type'],
                                      'hash': entry.content['hash']})
                    entrynodes.append(node)
                contents.append(E.entry(*list(entrynodes)))
            feed = E.feed(*contents)
            res = etree.tostring(feed,
                                 pretty_print=True,
                                 xml_declaration=True,
                                 encoding='utf-8')
            fileno, tmpfile = mkstemp()
            fp = os.fdopen(fileno)
            fp.close()
            # tmpfile = mkstemp()[1]
            with open(tmpfile, "wb") as fp:
                fp.write(res)
            util.replace_if_different(tmpfile, feedfile)
            return feedfile

        assert isinstance(entries, list), 'entries should be a list, not %s' % type(entries)
        feedurl = self.generic_url(basefile, 'feed', '.atom')
        # not sure abt this - should be uri of dataset?
        feedid = feedurl

        # assume entries are sorted newest first
        # could be simplified with more_itertools.chunked?
        cnt = 0
        res = []
        # print("chunking...")
        while len(entries) >= archivesize * 2:
            cnt += 1
            archiveentries = entries[-archivesize:]
            entries[:] = entries[:-archivesize]

            if cnt > 1:
                prev = "%s-archive-%s.atom" % (basefile, cnt - 1)
            else:
                prev = None
            if len(entries) < archivesize * 2:
                next = "%s.atom" % basefile
            else:
                next = "%s-archive-%s.atom" % (basefile, cnt + 1)
            suffix = suffix = '-archive-%s' % cnt
            res.append(write_file(archiveentries, suffix=suffix,
                                  prevarchive=prev,
                                  nextarchive=next))

        res.insert(0, write_file(entries,
                                 prevarchive="%s-archive-%s.atom" % (basefile, cnt)))
        return res

    def frontpage_content(self, primary=False):
        """If the module wants to provide any particular content on
        the frontpage, it can do so by returning a XHTML fragment (in
        text form) here. If primary is true, the caller wants the
        module to take primary responsibility for the frontpage
        content. If primary is false, the caller only expects a
        smaller amount of content (like a smaller presentation of the
        repository and the document it contains)."""
        g = self.make_graph()
        qname = g.qname(self.rdf_type)
        return ("<h2><a href='%s'>Document repository '%s'</a></h2><p>Handles %s documents. "
                "Contains %s published documents.</p>"
                % (self.dataset_uri(), self.alias, qname,
                   len(list(self.store.list_basefiles_for("_postgenerate")))))

    def status(self, basefile=None, samplesize=3):
        """Prints out some basic status information about this repository."""

        print("Status for document repository '%s' (%s)" %
              (self.alias, getattr(self.config, 'class')))
        s = self.get_status()
        for step in s.keys():  # odict
            exists = s[step]['exists']
            todo = s[step]['todo']
            exists_sample = ", ".join(exists[:samplesize])
            exists_more = len(exists) - samplesize
            todo_sample = ", ".join(todo[:samplesize])
            todo_more = len(todo) - samplesize

            if not exists_sample:
                exists_sample = "None"
            if exists_more > 0:
                exists_more_label = ".. (%s more)" % exists_more
            else:
                exists_more_label = ""

            if todo_more > 0:
                todo_more_label = ".. (%s more)" % todo_more
            else:
                todo_more_label = ""

            if step == 'download':
                print(" download: %s.%s" % (exists_sample, exists_more_label))
            else:
                if todo_sample:
                    print(" %s: %s.%s Todo: %s.%s" % (step, exists_sample, exists_more_label,
                                                      todo_sample, todo_more_label))
                else:
                    print(" %s: %s.%s" % (step, exists_sample, exists_more_label))

        # alias and classname
        # $ ./ferenda-build.py w3c status
        # Status for document repository 'w3c' (w3cstandards.W3Cstandards)
        # downloaded: rdb-direct-mapping r2rml ... (141 more)
        # parsed: None (143 needs parsing)
        # generated: None (143 needs generating)
    def get_status(self):
        """Returns basic data about the state about this repository, used by
        :meth:`~ferenda.DocumentRepository.status`.

        """
        status = OrderedDict()
        exists = []
        todo = []
        for basefile in self.store.list_basefiles_for("parse"):
            exists.append(basefile)
            # no point in trying to append
        status['download'] = {'exists': exists,
                              'todo': todo}

        # parse
        exists = []
        todo = []
        for basefile in self.store.list_basefiles_for("parse"):
            dependency = self.store.downloaded_path(basefile)
            target = self.store.parsed_path(basefile)
            if os.path.exists(target):
                exists.append(basefile)
            # Note: duplication of (part of) parseifneeded logic
            if not util.outfile_is_newer([dependency], target):
                todo.append(basefile)
        status['parse'] = {'exists': exists,
                           'todo': todo}

        # generated
        exists = []
        todo = []
        for basefile in self.store.list_basefiles_for("generate"):
            dependency = self.store.parsed_path(basefile)
            target = self.store.generated_path(basefile)
            if os.path.exists(target):
                exists.append(basefile)
            # Note: duplication (see above)
            if not util.outfile_is_newer([dependency], target):
                todo.append(basefile)
        status['generated'] = {'exists': exists,
                               'todo': todo}
        return status

    def tabs(self):
        """Get the navigation menu segment(s) provided by this docrepo

        Returns a list of tuples, where each tuple will be rendered
        as a tab in the main UI. First element of the tuple is the
        link text, and the second is the link destination. Normally, a
        module will only return a single tab.

        :returns: List of tuples
        """
        uri = self.dataset_uri()
        if self.rdf_type == Namespace(util.ns['foaf']).Document:
            return [(self.alias, uri)]
        else:
            return [(util.uri_leaf(str(self.rdf_type)), uri)]

    def footer(self):
        """Get a list of resources provided by this repo for publication in the site footer.

        Works like :meth:`~ferenda.DocumentRepository.tabs`, but
        normally returns an empty list. The repo
        :class:`ferenda.sources.general.Static` is an exception.
        """
        return []

    def http_handle(self, environ):
        """Used by the WSGI support to indicate if this repo can
        provide a response to a particular request. If so, returns a
        tuple (fp, length, memtype), where fp is an open file of the document to be returned."""
        if environ['PATH_INFO'].count("/") >= 2:
            segments = environ['PATH_INFO'].split("/", 3)
            if len(segments) == 3:
                null, res, alias = segments
            else:
                null, res, alias, basefile = segments

            if (alias == self.alias):
                # we SHOULD be able to handle this -- maybe provide
                # apologetic message about this if we can't?
                uri = request_uri(environ)
                path = None
                
                accept = environ.get('HTTP_ACCEPT', 'text/html')
                # do proper content-negotiation, but make sure
                # application/xhtml+xml ISN'T one of the
                # available options (as modern browsers may
                # prefer it to text/html, and our
                # application/xhtml+xml isn't what they want)
                # -- ie we only serve application/xtml+xml if
                # a client specifically only asks for
                # that. Yep, that's a big FIXME.
                available = ("text/html")  # add to this?
                preferred = httpheader.acceptable_content_type(accept,
                                                               available)

                rdfformats = {'application/rdf+xml': 'pretty-xml',
                              'text/turtle': 'turtle',
                              'text/plain': 'nt'}
                
                if res == "res":
                    if uri.endswith("/data"):
                        data = True
                        uri = uri[:-5]
                    else:
                        data = False
                    basefile = self.basefile_from_uri(uri)
                    assert basefile, "Couldn't find basefile in uri %s" % uri

                    # mapping MIME-type -> callable that retrieves a path
                    pathfunc = None
                    if not data:
                        pathmap = {'text/html': self.store.generated_path,
                                   'application/xhtml+xml': self.store.parsed_path,
                                   'application/rdf+xml': self.store.distilled_path}
                        if accept in pathmap:
                            contenttype = accept
                            pathfunc = pathmap[accept]
                        else:
                            if preferred and preferred[0].media_type == "text/html":
                                contenttype = preferred[0].media_type
                                pathfunc = self.store.generated_path

                    if pathfunc is None:
                        if accept in rdfformats:
                            contenttype = accept
                            g = Graph()
                            g.parse(self.store.distilled_path(basefile))
                            if data:
                                annotation_graph = self.annotation_file_to_graph(
                                    self.store.annotation_path(basefile))
                                g += annotation_graph
                            path = None
                            # FIXME: we just changed the meaning of the "data" variable!
                            data = g.serialize(format=rdfformats[accept])
                        else:
                            data = None
                    else:
                        path = pathfunc(basefile)
                        data = None
                elif res == "dataset":
                    # FIXME: this reimplements the logic that
                    # calculates basefile/path at the end of
                    # toc_pagesets AND transform_links
                    contenttype = accept
                    if preferred and preferred[0].media_type == "text/html":
                        contenttype = preferred[0].media_type

                    if contenttype == "text/html":
                        params = self.dataset_params_from_uri(uri)
                        if params:
                            pseudobasefile = "/".join(params)
                        else:
                            pseudobasefile = "index"
                        path = self.store.path(pseudobasefile, 'toc', '.html')
                        contenttype = "text/html"
                    elif contenttype == "text/plain":
                        path = self.store.path("dump", "distilled", ".nt")
                    elif contenttype in rdfformats:
                        g = Graph()
                        g.parse(self.store.path("dump", "distilled", ".nt"),
                                format="nt")
                        data = g.serialize(format=rdfformats[accept])

                if path and os.path.exists(path):
                    return (open(path, 'rb'),
                            os.path.getsize(path),
                            200,
                            contenttype)
                elif data:
                    return (BytesIO(data),
                            len(data),
                            200,
                            contenttype)
                else:
                    msg = "<h1>406</h1>No acceptable media found for <tt>%s</tt>" % accept
                    return(BytesIO(msg.encode('utf-8')),
                           len(msg.encode('utf-8')),
                           406,
                           "text/html")

        return (None, None, None, None)

    @staticmethod
    def _setup_logger(logname):
        log = logging.getLogger(logname)
        if log.handlers == []:
            if hasattr(logging, 'NullHandler'):
                log.addHandler(logging.NullHandler())
            else:  # pragma: no cover
                # py26 compatibility
                class NullHandler(logging.Handler): 

                    def emit(self, record):
                        pass
                log.addHandler(NullHandler())
        return log
