"""
downstream parser for pygetpapers
"""
import datetime
import json
from collections import defaultdict
from pathlib import Path
from re import RegexFlag

import lxml
import lxml.etree as ET
from lxml.etree import _Element
from lxml.html import HTMLParser, HtmlElement

from amilib.ami_bib import EUPMC_RESULTS_JSON, EUPMC_TRANSFORM
from amilib.ami_html import HtmlLib, HtmlUtil, Datatables
from amilib.ami_util import AmiJson, AmiUtil
from amilib.file_lib import FileLib
from amilib.util import Util, TextUtil

DATATABLES_HTML = "datatables.html"
SAVED_CONFIG_INI = "saved_config.ini"  # TODO cyclic import
HTML_WITH_IDS = "html_with_ids"

logger = Util.get_logger(__name__)







class AmiCorpus():
    """
    supports a tree of directories and leaf documents with wrappers AmiCorpus
    (top dir) and AmiCorpusComponents (subdirectories)

    """
    TABLE_MARKUP_HITS = "table_markup_hits"

    def __init__(self,
                 topdir=None,
                 infiles=None,
                 globstr=None,
                 outfile=None,
                 make_descendants=False,
                 mkdir=False,
                 **kwargs):
        """
        create new Corpus, withn optional input of data
        :param topdir: Input directory with files/subdirs as possible corpus components
        :param infiles: list of files to use (alternative to globstr)
        :param globstr: create infiles using globbing under topdir (requires topdir)
        :param outfile:
        :param mkdir: make topdir if doesn't exist (default=False)
        :param make_descendants: makes AmiCorpusContianers for directories on tree
        :param kwargs: dict of per-corpus user-specified properties

        """
        self.topdir = topdir
        if self.topdir and not self.topdir.is_dir():
            raise ValueError(f"AmiCorpus() requires valid directory {self.topdir}")

        self.container_by_file = dict()
        # rootnode
        self.ami_container = self.create_corpus_container(
            self.topdir, make_descendants=make_descendants, mkdir=mkdir)
        self.infiles = infiles
        self.outfile = outfile
        self.globstr = globstr
        self.corpus_queries = dict()
        self.search_html = None
        self.eupmc_results = None
        self._make_infiles()
        self._make_outfile()

        self.make_special(kwargs)

    def __str__(self):
        values = []
        for key in self.corpus_queries.keys():
            value = self.corpus_queries.get(key)
            values.append(value)


            s = f"""AmiCorpus:
topdir:  {self.topdir}
globstr: {self.globstr}
infiles: {len(self.infiles)}
outfile: {self.outfile}
"""
            s += "QUERIES"
            for (key, value) in self.corpus_queries.items():
                s += f"""
                {key}:
                {value.__str__()}
                """
        return s

    def _make_infiles(self):
        if self.infiles:
            logger.info(f"taking infiles from list")
        else:
            if self.topdir and self.globstr:
                self.infiles = FileLib.posix_glob(f"{self.topdir}/{self.globstr}", recursive=True)
        if self.infiles is None:
            logger.error(f"self.infiles is None")
            return
        logger.info(f"inputting {len(self.infiles)} files")
        return self.infiles

    def create_corpus_container(self, file, bib_type="None", make_descendants=False, mkdir=False):
        """
        create container as child of self
        :param file: file or dir contained by container

        """
        if file is None:
            logger.error("Container has no file")
            return None

        container = AmiCorpusContainer(self, file)
        container.bib_type = bib_type
        if not file.exists() and mkdir:
            Path(file).mkdir()
        if make_descendants:
            self.make_descendants(file)
        return container

    @property
    def root_dir(self):
        return self.ami_container.file

    def make_descendants(self, file=None):
        """
        creates AmiCorpusContainers for directory tree
        :param ami_corpus: top-level corpus or None
        :param parent_dir: top-level directory of corpus
        :param source_dir: each directory in tree
        :return; None
        """
        if file is None:
            file = self.root_dir
        if file is None or not file.is_dir():
            logger.error(f"Cannot make file children for {file}")
            return
        files = FileLib.get_children(file)
        for f in files:
            container = AmiCorpusContainer(self, f)
            container.make_descendants()

    def make_special(self, kwargs):
        """manges repository specific stuff (e.g. for EUPMC)"""
        """Horrible - use dependency injectiom
        """
        if "eupmc" in kwargs:
            self.eupmc_results = kwargs["eupmc"]

    def get_datatables_from_eupmc_results(self):
        """
        deprecated
        create a JQuery datatables from eumpc_reults.json
        TODO rewrite - EUPMC should not be in here
        """
        self.get_or_create_eupmc_results()
        if self.eupmc_results is None:
            logger.error(f"no {EUPMC_RESULTS_JSON}, so cannot create datatables at present")
            return

    def get_or_create_eupmc_results(self):
        f"""
        gets file {EUPMC_RESULTS_JSON} from directory
        :return: {EUPMC_RESULTS_JSON} from directory or None if not exists
        """
        if self.eumpc_results is None:
            if not Path(self.source_dir).is_dir():
                logger.error(f"not a directory {self.source_dir}")
                return None
            self.eupmc_results = Path(self.eupmc_results, EUPMC_RESULTS_JSON)
            if not self.eupmc_results.exists():
                self.eupmc_results = None
        return self.eupmc_results

    @classmethod
    def make_datatables(cls, indir, outdir=None, outfile_h=None):
        """
        creates a JQuery.datatables (HTML) file from an AmiCorpus
        May make this method a member function later
        :param indir: directory with corpus (normally created by pygetpapers)
        :param outdir: output for datatables (if omitted uses indir)
        :param outfile_h: the HTML file created (may be changed)
        """
        from amilib.ami_bib import SAVED_CONFIG_INI

        if indir is None:
            logger.warning("No indir")
            return
        if outdir is None:
            outdir = indir
        if outfile_h is None:
            outfile_h = Path(outdir, DATATABLES_HTML)
        # wanted_keys = [PMCID, DOI, TITLE, AUTHOR_STRING, JOURNAL_INFO_TITLE, PUB_YEAR, ABS_TEXT]
        config_ini = Path(indir, SAVED_CONFIG_INI)
        outdir.mkdir(parents=True, exist_ok=True)
        datatables = True
        epmc_infile = Path(indir, EUPMC_RESULTS_JSON)
        openalex_infile = Path(indir, OpenAlex.RESULTS_JSON)
        if epmc_infile.exists():

            AmiCorpus.read_json_create_write_html_table(
                epmc_infile, outfile_h, wanted_keys=None, datatables=datatables,
                table_id=None, config_ini=config_ini)
            return
        infile = Path(indir, EUPMC_RESULTS_JSON)
        if infile.exists():
            AmiCorpus.read_json_create_write_html_table(
                infile, outfile_h, wanted_keys=None, datatables=datatables, table_id=None, config_ini=config_ini)
            return

    @classmethod
    def read_json_create_write_html_table(
            cls, infile, outfile_h, wanted_keys,
            styles=None, datatables=None, table_id=None, config_ini=None):
        """
        read pygetpapers output, select columns and create HTML table
        :param infile: eumpc_results from pygetpapers
        :param outfile_h: HTML table to write
        :param wanted_keys: column headings to select
        :param styles: datatables styles
        :param datatables: datatables
        :param table_id: id links script to table (default "my_table")
        :param config_ini: adds info from saved_config.ini - mainly the query
        :return: html table
        """
        # TODO cyclic imports
        from amilib.ami_bib import PMCID, TITLE
        from amilib.ami_bib import DOI, AUTHOR_STRING, JOURNAL_INFO_TITLE, PUB_YEAR, ABS_TEXT
        if styles is None:
            styles = ["td {border:solid 1px black;}"]
        if table_id is None:
            table_id = "my_table"
        if wanted_keys is None:
            wanted_keys = [PMCID, DOI, TITLE, AUTHOR_STRING, JOURNAL_INFO_TITLE, PUB_YEAR, ABS_TEXT]

        assert infile.exists(), f"infile {infile} must exist"
        with open(infile, "r") as f:
            jsonx = json.load(f)
        # look for all papers
        papers = jsonx.get("papers")
        assert papers is not None, f"cannot find papers"
        # specific keys we want
        dict_by_id = AmiJson.create_json_table(papers, wanted_keys)
        htmlx, table = HtmlLib.create_html_table(
            dict_by_id, transform_dict=EUPMC_TRANSFORM,
            styles=styles, datatables=datatables, table_id=table_id
        )
        cls.add_query_as_caption(config_ini, table)

        HtmlUtil.write_html_elem(htmlx, outfile_h, debug=True)

    @classmethod
    def add_query_as_caption(cls, config_ini, table):
        """
        add query string from config.ini to table caption
        :param config_ini: saved_config.ini file
        :param table: to add caption to
        """
        from amilib.ami_bib import SAVED, QUERY, STARTDATE, ENDDATE
        if config_ini is not None:
            config, _ = AmiUtil.get_config_and_section_names(config_ini)
            saved_section = config[SAVED]
            query = saved_section[QUERY]
            startdate = saved_section[STARTDATE]
            if startdate == 'False':
                startdate = "..."
            enddate = saved_section[ENDDATE]
            if enddate == 'False':
                enddate = datetime.date.today()

            caption = ET.SubElement(table, "caption")
            caption.text = f"query: {query}"
            if startdate:
                caption.text += f"; start: {startdate}"
            if enddate:
                caption.text += f"; end: {enddate}"

    @classmethod
    def search_files_with_phrases_write_results(cls, infiles, phrases=None, para_xpath=None, outfile=None, debug=False):
        """
        iterates over files in corpus
        """
        if phrases is None:
            logger.error("no phrases")
            return
        all_hits_dict = dict()
        url_list_by_phrase_dict = defaultdict(list)
        if (type(phrases) is not list):
            phrases = [phrases]
        all_paras = []
        for infile in infiles:
            paras = cls.search_paras_with_id_and_create_dict(all_hits_dict, infile, para_xpath, phrases,
                                                     url_list_by_phrase_dict)
            all_paras.extend(paras)

        if debug:
            print(f"para count~: {len(all_paras)}")
        html1 = cls.create_html_from_hit_dict(url_list_by_phrase_dict)
        assert html1 is not None
        if outfile:
            outfile = Path(outfile)
            outfile.parent.mkdir(exist_ok=True, parents=True)
            with open(outfile, "w", encoding="UTF-8") as f:
                if debug:
                    print(f" hitdict {url_list_by_phrase_dict}")
                HtmlLib.write_html_file(html1, outfile, debug=True)
        return html1

    @classmethod
    def search_paras_with_id_and_create_dict(cls, all_hits_dict, infile, para_xpath, phrases,
                                             url_list_by_phrase_dict):
        """
        should be instance method
        reads file, creates HTML, finds paras_with_id,
        """

        assert Path(infile).exists(), f"{infile} does not exist"
        html_tree = lxml.etree.parse(str(infile), HTMLParser())
        paras = HtmlLib.find_paras_with_ids(html_tree, para_xpath=para_xpath)
        # this does the search
        para_id_by_phrase_dict = HtmlLib.create_search_results_para_phrase_dict(paras, phrases)
        if para_id_by_phrase_dict is not None and len(para_id_by_phrase_dict) > 0:
            cls.add_hit_with_filename_and_para_id(all_hits_dict, url_list_by_phrase_dict, infile,
                                                  para_id_by_phrase_dict)
        return paras

    @classmethod
    def add_hit_with_filename_and_para_id(cls, all_hits_dict, hit_dict, infile, phrase_by_para_id_dict):
        """adds non-empty hits in hit_dict and all to all_dict
        :param all_hits_dict: accumulates para_phrase_dict by infile
        :param hit_dict: accumulates url by hit (

        """
        infile_s = cls.create_url_from_filename(infile)

        # what is item[1] ???
        item_paras = [item for item in phrase_by_para_id_dict.items() if len(item[1]) > 0]
        if len(item_paras) > 0:
            all_hits_dict[infile] = phrase_by_para_id_dict
            for para_id, hits in phrase_by_para_id_dict.items():
                for hit in hits:
                    url = f"{infile_s}#{para_id}"
                    hit_dict[hit].append(url)
                    # logger.info(f"hit: {hit} in url {url}")

    @classmethod
    def create_url_from_filename(cls, infile):
        # TODO should write file with slashes (on Windows we get %5C)
        infile_s = f"{infile}"
        infile_s = infile_s.replace("\\", "/")
        infile_s = infile_s.replace("%5C", "/")
        return infile_s

    @classmethod
    def add_hit_with_filename_and_para_id(cls, all_dict, hit_dict, infile, para_phrase_dict):
        """adds non-empty hits in hit_dict and all to all_dict
        :param all_dict: accumulates para_phrase_dict by infile

        TODO - move to amilib
        """
        infile_s = f"{infile}"
        infile_s = infile_s.replace("\\", "/")
        infile_s = infile_s.replace("%5C", "/")

        item_paras = [item for item in para_phrase_dict.items() if len(item[1]) > 0]
        if len(item_paras) > 0:
            all_dict[infile] = para_phrase_dict
            for term, para_id in para_phrase_dict.items():
                # TODO should write file with slashes (on Windows we get %5C)
                url = f"{infile_s}#{para_id}"
                hit_dict[term].append(url)

    @classmethod
    def create_html_from_hit_dict(cls, hit_dict):
        """
        This looks awful
        """
        html = HtmlLib.create_html_with_empty_head_body()
        body = HtmlLib.get_body(html)
        ul = ET.SubElement(body, "ul")
        for term, hits in hit_dict.items():
            li = ET.SubElement(ul, "li")
            p = ET.SubElement(li, "p")
            p.text = f"term: {term}"
            ul1 = ET.SubElement(li, "ul")
            for hit in hits:
                # TODO manage hits with Paths
                # on windows some hits have "%5C' instead of "/"
                hit = str(hit).replace("%5C", "/")
                li1 = ET.SubElement(ul1, "li")
                a = ET.SubElement(li1, "a")
                a.text = hit.replace("/{HTML_WITH_IDS}.html", "")
                ss = "ipcc/"
                try:
                    idx = a.text.index(ss)
                except Exception as e:
                    print(f"cannot find substring {ss} in {a.text}")
                    continue
                a.text = a.text[idx + len(ss):]
                a.attrib["href"] = hit
        return html


    @classmethod
    def add_content_for_files(cls, files, tr):
        if files:
            HtmlLib.add_cell_content(tr, text=Path(files[0]).stem, href=f"file://{files[0]}")
        else:
            HtmlLib.add_cell_content(tr, text="?")

    def list_files(self, globstr):
        """
        finds files in corpus starting at root_dir
        :param globstr:
        """
        if globstr and self.root_dir:
            return FileLib.list_files(self.root_dir, globstr=globstr)
        return []

    def create_datatables_html_with_filenames(self, html_glob, labels, table_id, outpath=None, debug=True):
        """
        :param html_glob: globstring to find html files
        :labels
        """
        html_files = sorted(self.list_files(globstr=html_glob))
        self.datables_html, tbody = Datatables._create_html_for_datatables(labels, table_id)

        for html_file in html_files:
            if outpath:
                # before python method
                offset = FileLib.get_relative_path(html_file, outpath.parent, walk_up=True)
                # offset = Path(html_file).relative_to(Path(outpath).parent)
            tr = ET.SubElement(tbody, "tr")
            HtmlLib.add_cell_content(tr, text=offset, href=f"{offset}")
        HtmlLib.write_html_file(self.datables_html, outpath, debug=debug)

        return self.datables_html

    def make_infiles(self, maxfiles=999999999):
        self.infiles = FileLib.posix_glob(self.globstr, recursive=True)[:maxfiles]
        return self.infiles


    def make_globstr(self):
        pass

    def _make_outfile(self):
        if not self.outfile:
        #     if self.query:
            pass

    def search_files_with_phrases(self, phrases, debug=False):
        """ TODO rewrite with AmiQuery"""
        if phrases:
            self.search_html = AmiCorpus.search_files_with_phrases_write_results(
                self.infiles, phrases=phrases, outfile=self.outfile, debug=debug)

    def get_or_create_corpus_query(self,
                                   query_id,
                                   phrasefile=None,
                                   phrases=None,
                                   outfile=None):
        """
        retrieves query by query_stem. If not found creates new one
        :param query_id: unique id of query
        :param outfile: file to output results of search
        :return: query
        """
        corpus_query = self.corpus_queries.get(query_id)
        if not corpus_query:
            corpus_query = CorpusQuery(
                query_id=query_id,
                phrasefile=phrasefile,
                phrases=phrases,
                outfile=outfile)
            self.corpus_queries[query_id] = corpus_query
            corpus_query.corpus = self

        return corpus_query

    def search_files_with_queries(self, query_ids: list[str], debug: bool =True) -> dict():
        """
        run queries. Assumed that queries have been loaded and recallable by id
        :param query_ids: single or list of ids
        """
        html_by_query_id = dict()
        if type(query_ids) is str:
            query_ids = [query_ids]
        elif (t :=type(query_ids)) is not list:
            raise ValueError(f"queries requires id/s as list or str, found {t}")
        for query_id in query_ids:
            query = self.corpus_queries.get(query_id)
            if query is None:
                logger.error(f"cannot find query: {query_id}")
                continue
            logger.debug(f"outfile==> {query.outfile}")
            self.search_html = AmiCorpus.search_files_with_phrases_write_results(
                self.infiles, phrases=query.phrases, outfile=query.outfile, debug=debug)

            term_id_by_url = CorpusQuery.make_hits_by_url(self.search_html)
            term_ref_p_tuple_list = CorpusQuery.get_hits_as_term_ref_p_tuple_list(term_id_by_url)
            htmlx, tbody = HtmlLib.make_skeleton_table(colheads=["term", "ref", "para"])
            CorpusQuery._add_hits_to_table(tbody, term_ref_p_tuple_list)
            html_by_query_id[query_id] = htmlx

        return html_by_query_id
            # AmiCorpus.search_files_with_phrases_write_results(self.infiles, phrases=query.phrases)

TABLE_HITS_SUFFIX = "table_hits.html"
class CorpusQuery:
    """
    holds query and related info (hits, files)
    """
    def __init__(self,
                 query_id=None,
                 phrasefile=None,
                 phrases=None,
                 outfile=None,
                 ):
        self.query_id = query_id
        self.phrasefile = phrasefile
        self.phrases = phrases
        if self.phrasefile and not self.phrases:
            self.phrases = FileLib.read_strings_from_path(self.phrasefile)

        self.outfile = outfile
        self.corpus = None
        logger.debug(f"made query {self}")

    def run_query_make_table_TUTORIAL(
            self, query: str | list[str], query_id:str, indir, outdir, outfile=None):
        """
        takes query string and query_id, creates input and output filenames
        runs query
        creates table of results
        :param query: query string or list of query strings
        :param query_id: string without spaces to uniquely identify the query
        :param indir: top directory of corpus
        :param outdir: output top directory
        :param outfile: output HTML file with tables
        """

        self.debug = True
        if not query:
            raise ValueError("No query given")
        if type(query) is str:
            self.phrases = list(query)
        if not query_id :
            raise ValueError("No query_id given")
        query_id = query_id.strip()
        if " " in query_id:
            raise ValueError(f"no spaces allowed in query_id, found {query_id}")
        self.query_id = query_id

        if indir is None or not Path(indir).exists():
            logger.error(f"input directory must exist {indir}")
        self.indir = indir
        self.outfile = outfile
        if not outfile:
            self.outfile = Path(self.indir, f"{query_id}.html")

        self.para_xpath = None

        self.globstr = f"{str(self.indir)}/**/{HTML_WITH_IDS}.html"
        self.infiles = FileLib.posix_glob(self.globstr, recursive=True)
        # assert 50 == len(infiles)
        self.phrases = ["methane emissions"]
        self.colheads = ["term", "ref", "para"]

        html_markup = AmiCorpus.search_files_with_phrases_write_results(
            self.infiles, phrases=self.phrases, para_xpath=self.para_xpath,
            outfile=self.outfile, debug=self.debug)
        term_id_by_url = CorpusQuery.extract_hits_by_url_from_nested_lists(html_markup)
        term_ref_p_tuple_list = CorpusQuery.get_hits_as_term_ref_p_tuple_list(term_id_by_url)
        htmlx, table_body = HtmlLib.make_skeleton_table(colheads=self.colheads)
        CorpusQuery._add_hits_to_table(table_body, term_ref_p_tuple_list)
        table_file = Path(outdir, f"{self.query_id}_{TABLE_HITS_SUFFIX}")
        HtmlLib.write_html_file(htmlx, table_file, debug=True)
        htmlx, table_body = HtmlLib.make_skeleton_table(colheads=self.colheads)
        new_term_ref_p_list = []
        for (term, ref, para) in term_ref_p_tuple_list:
            has_markup = HtmlLib.find_and_markup_phrases(
                para, term, ignore_case=True, markup=True, flags=RegexFlag.IGNORECASE)
            new_term_ref_p_list.append((term, ref, para))
        CorpusQuery._add_hits_to_table(table_body, new_term_ref_p_list)
        return htmlx, self.query_id

    def __str__(self):
        s = f"""CorpusQuery
query_id:  {self.query_id}
phrasefile:{self.phrasefile}
phrases:   {None if not self.phrases else len(self.phrases) }
outfile:   {self.outfile}
corpus:    {None if self.corpus is None else self.corpus.__hash__() }
"""
        return s
    @classmethod
    def make_hits_by_url(cls, nested_list_html):
        """
        <body>
          <ul>
            <li>term: xyz
              <li><a href-to-para
        """
        # iterate over hit list
        if nested_list_html is None:
            logger.error(f"html1 is None")
            return None
        body = HtmlLib.get_body(nested_list_html)
        query_ul = HtmlLib.get_first_object_by_xpath(body, "ul")
        hits_by_url = dict()
        for li in query_ul.xpath("li"):
            # logger.debug("li")
            p0 = HtmlLib.get_first_object_by_xpath(li, "p")
            if p0 is None:
                continue
            term = p0.text
            txt = "term: "
            if (term.startswith(txt)):
                term = term[len(txt):]
            hits_ul = HtmlLib.get_first_object_by_xpath(li, "ul")
            if hits_ul is None:
                continue
            hits_li_list = hits_ul.xpath("li")
            # logger.debug(f"hits {len(hits_li_list)}")
            for hits_li in hits_li_list:
                # logger.debug(f"hits_li")
                cls.add_hit_list_to_hits_by_url(hits_by_url, hits_li, term)
                # logger.debug(f"added hits_li")

        return hits_by_url


    @classmethod
    def add_hit_list_to_hits_by_url(cls, hits_by_url, hits_li, term):
        if hits_by_url is None or hits_li is None or term is None:
            logger.error(f"add_hit_list None args ")
            return
        a = HtmlLib.get_first_object_by_xpath(hits_li, "a")
        if a is None:
            logger.error(f"a is None")
            return
        # logger.debug(f"a is {a}")
        href = a.attrib.get("href")
        if href is None:
            logger.error(f"href is None {ET.tostring(a)}")
            return
        # logger.debug(f"href {href}")
        href_target = href.split("#")[0]
        id = href.split("#")[1]
        # logger.debug(f"href_target {href_target}")
        html_targ = HtmlLib.parse_html(href_target)
        assert html_targ is not None
        if "[" in id:
            # logger.debug("quoted list")
            id_list = TextUtil.convert_quoted_list_to_list(id)
            for id1 in id_list:
                cls._get_element_by_id_and_add_term_id_tuple_to_hits(hits_by_url, href_target, html_targ, id1, term)
        else:
            # logger.debug("non quoted list")
            cls._get_element_by_id_and_add_term_id_tuple_to_hits(hits_by_url, href_target, html_targ, id, term)
        # logger.debug("exit add hitlist")


    @classmethod
    def _get_element_by_id_and_add_term_id_tuple_to_hits(cls, hits_by_url, href_target, html_targ, id, term):
        if hits_by_url is None or href_target is None or html_targ is None or id is None or term is None:
            logger.error("arg is None")
            return None
        p = HtmlLib.get_element_by_id(html_targ, id)
        if p is not None:
            tuple = (term, p)
            target_id = f"{href_target}#{id}"
            hits_by_url[target_id] = (tuple)
        # logger.debug("exit _get_element_by_id_and_add_term_id_tuple_to_hits")

    @classmethod
    def get_hits_as_term_ref_p_tuple_list(cls, term_id_by_url):
        if term_id_by_url is None:
            logger.error(f"term_id_by_url is None")
            return None
        trp_list = []
        for ref in term_id_by_url.keys():
            bits = ref.split("#")
            file = bits[0]
            idref = bits[1]
            term_p = term_id_by_url.get(ref)
            term = term_p[0]
            p = term_p[1]
            # logger.debug(f"{term}:{idref} => {''.join(p.itertext())}")
            tuple = (term, ref, p)
            trp_list.append(tuple)

        return trp_list

    @classmethod
    def _add_hits_to_table(cls, tbody, term_ref_p_tuple_list):
        if term_ref_p_tuple_list is None:
            logger.error(f"term_ref_p_tuple_list is None")
            return
        for term, ref, p in term_ref_p_tuple_list:
            assert type(term) is str
            assert type(ref) is str
            assert type(p) is _Element or type(p) is HtmlElement
            tr = ET.SubElement(tbody, "tr")
            tds = []
            for item in term, ref, p:
                tds.append(ET.SubElement(tr, "td"))
            tds[0].text = term
            a = ET.SubElement(tds[1], "a")
            a.attrib["href"] = ref
            a.text = ref

            tds[2].append(p)

    @classmethod
    def extract_hits_by_url_from_nested_lists(cls, nested_list_html):
        """
        <body>
          <ul>
            <li>term: xyz
              <li><a href-to-para
        """
        # iterate over hit list
        if nested_list_html is None:
            logger.error(f"html1 is None")
            return None
        body = HtmlLib.get_body(nested_list_html)
        query_ul = HtmlLib.get_first_object_by_xpath(body, "ul")
        hits_by_url = dict()
        for li in query_ul.xpath("li"):
            # logger.debug("li")
            p0 = HtmlLib.get_first_object_by_xpath(li, "p")
            if p0 is None:
                continue
            term = p0.text
            txt = "term: "
            if (term.startswith(txt)):
                term = term[len(txt):]
            hits_ul = HtmlLib.get_first_object_by_xpath(li, "ul")
            if hits_ul is None:
                continue
            hits_li_list = hits_ul.xpath("li")
            # logger.debug(f"hits {len(hits_li_list)}")
            for hits_li in hits_li_list:
                # logger.debug(f"hits_li")
                CorpusQuery.add_hit_list_to_hits_by_url(hits_by_url, hits_li, term)
                # logger.debug(f"added hits_li")

        return hits_by_url

    @classmethod
    def add_hit_list_to_hits_by_url(cls, hits_by_url, hits_li, term):
        if hits_by_url is None or hits_li is None or term is None:
            logger.error(f"add_hit_list None args ")
            return
        a = HtmlLib.get_first_object_by_xpath(hits_li, "a")
        if a is None:
            logger.error(f"a is None")
            return
        # logger.debug(f"a is {a}")
        href = a.attrib.get("href")
        if href is None:
            logger.error(f"href is None {ET.tostring(a)}")
            return
        # logger.debug(f"href {href}")
        href_target = href.split("#")[0]
        id = href.split("#")[1]
        # logger.debug(f"href_target {href_target}")
        html_targ = HtmlLib.parse_html(href_target)
        assert html_targ is not None
        if "[" in id:
            # logger.debug("quoted list")
            id_list = TextUtil.convert_quoted_list_to_list(id)
            for id1 in id_list:
                CorpusQuery._get_element_by_id_and_add_term_id_tuple_to_hits(hits_by_url, href_target, html_targ, id1, term)
        else:
            # logger.debug("non quoted list")
            CorpusQuery._get_element_by_id_and_add_term_id_tuple_to_hits(hits_by_url, href_target, html_targ, id, term)
        # logger.debug("exit add hitlist")

    @classmethod
    def _get_element_by_id_and_add_term_id_tuple_to_hits(cls, hits_by_url, href_target, html_targ, id, term):
        if hits_by_url is None or href_target is None or html_targ is None or id is None or term is None:
            logger.error("arg is None")
            return None
        p = HtmlLib.get_element_by_id(html_targ, id)
        if p is not None:
            tuple = (term, p)
            target_id = f"{href_target}#{id}"
            hits_by_url[target_id] = (tuple)
        # logger.debug("exit _get_element_by_id_and_add_term_id_tuple_to_hits")




class AmiCorpusContainer:
    def __init__(self, ami_corpus, file, bib_type="unknown", mkdir=False, exist_ok=True):
        """
        create corpusContainer for directory-structured corpus
        :param ami_corpus: corpus to which this belonds
        :param file_node: file/dir on filesystem
        :param bib_type: type of container (e.g. report
        """
        if (t := type(ami_corpus)) is not AmiCorpus:
            raise ValueError(f"ami_corpus has wrong type {t}")
        self.ami_corpus = ami_corpus
        self.file = file
        self.ami_corpus.container_by_file[self.file] = self
        if not file:
            logger.error(f"No file_node argument")
            return None
        # if mkdir and self.source_dir:
        #     Path(self.source_dir).mkdir(exist_ok=exist_ok)
        self.bib_type = bib_type
        self.child_container_list = []
        self.child_document_list = []

    @property
    def child_containers(self):
        child_containers = []
        if self.ami_corpus and self.ami_corpus.source_dir:
            child_nodes = FileLib.get_children()
            for child_node in child_nodes:
                child_container = AmiCorpusContainer(self.ami_corpus, child_node)
                child_container.type = "" if Path(child_node).is_dir() else "file"
                child_containers.append(child_container)
        return child_containers

    def create_corpus_container(self, filename, bib_type="unknown", make_descendants=False, mkdir=False):
        """
        creates a child container and optionally its actual directory
        """
        if not filename :
            logger.error("filename is None")
            return None
        path = Path(self.file, filename)
        if mkdir and not path.exists():
            path.mkdir()
        else:
            if not path.is_dir():
                logger.error(f"{path} exists but is not a directory")
                return None
        corpus_container = AmiCorpusContainer(self.ami_corpus, path, bib_type=bib_type, mkdir=mkdir)
        self.child_container_list.append(corpus_container)
        return corpus_container

    def create_document(self, filename, text=None, type="unknown"):
        """
        creates document file with name and self as parent
        """
        document_file = Path(self.file, filename)
        document_file.touch()
        if text:
            with open(document_file, "w", encoding="UTF-8") as f:
                f.write(text)

        self.child_document_list.append(document_file)
        return document_file

    def make_descendants(self):
        if (self.file and self.file.is_dir()):
            self.ami_corpus.make_descendants(self.file)

# to create a hierarchical structure for your Corpus element in Python, you'll need to define a few classes that represent the Corpus, Container, and their relationships. Here's a simple implementation that achieves this:
#
# python
# Copy code
class Container:
    def __init__(self, name):
        self.name = name
        self.children = []
        self.parent = None

    def add_child(self, child):
        child.parent = self
        self.children.append(child)

    def __repr__(self):
        return f"Container({self.name}, children={len(self.children)})"


class Corpus(Container):
    def __init__(self, directory):
        super().__init__(directory)
        # Assume directory is the top-level directory wrapped by this Corpus
        self.directory = directory

    def __repr__(self):
        return f"Corpus({self.directory})"


# Example Usage
if __name__ == "__main__":
    # Create a Corpus for the top directory
    root_corpus = Corpus("root_directory")

    # Create Containers for subdirectories/files
    sub_container1 = Container("sub_directory1")
    sub_container2 = Container("sub_directory2")

    # Add containers as children to the root corpus
    root_corpus.add_child(sub_container1)
    root_corpus.add_child(sub_container2)

    # You can further add children to the sub_containers
    file1 = Container("file1.txt")
    sub_container1.add_child(file1)

    # Display the structure
    print(root_corpus)
    for child in root_corpus.children:
        print(f"  - {child}")
        for grandchild in child.children:
            print(f"    - {grandchild}")
# Explanation
# Container Class:
#
# Holds a name, a list of children, and a reference to its parent.
# The add_child method adds a child and sets its parent to the current container.
# Corpus Class:
#
# Inherits from Container and represents the top-level directory (or corpus).
# It takes a directory name during initialization.
# Example Usage:
#
# Creates a Corpus for the top directory.
# Adds child Container elements (subdirectories or files).
# Prints the structure of the corpus, including its children.
# Usage
# You can expand this structure by adding methods for traversing, searching,
# or manipulating the corpus as needed.
# Each Container instance can easily access its parent and children,
# allowing you to manage the hierarchical structure effectively.


class OpenAlex:
    """
    Trimmed JSON example
    {
	"total_json_output": {
		"10.1126_science.287.5452.443": {
			"id": "https://openalex.org/W2150891147",
			"doi": "https://doi.org/10.1126/science.287.5452.443",
			"title": "Emerging Infectious Diseases of Wildlife-- Threats to Biodiversity and Human Health",
			"display_name": "Emerging Infectious Diseases of Wildlife-- Threats to Biodiversity and Human Health",
			"relevance_score": 1916.973,
			"publication_year": 2000,
			"publication_date": "2000-01-21",
			"ids": {
				"openalex": "https://openalex.org/W2150891147",
				"doi": "https://doi.org/10.1126/science.287.5452.443",
				"mag": "2150891147",
				"pmid": "https://pubmed.ncbi.nlm.nih.gov/10642539"
			},
			"language": "en",
			"primary_location": {
				"is_oa": false,
				"landing_page_url": "https://doi.org/10.1126/science.287.5452.443",
				"pdf_url": null,
				"source": {
					"id": "https://openalex.org/S3880285",
					"display_name": "Science",
					"issn_l": "0036-8075",
					"issn": [
						"0036-8075",
						"1095-9203"
					],
					"is_oa": false,
					"is_in_doaj": false,
					"is_indexed_in_scopus": true,
					"is_core": true,
					"host_organization": "https://openalex.org/P4310315823",
					"host_organization_name": "American Association for the Advancement of Science",
					"host_organization_lineage": [
						"https://openalex.org/P4310315823"
					],
					"host_organization_lineage_names": [
						"American Association for the Advancement of Science"
					],
					"type": "journal"
				},
				"license": null,
				"license_id": null,
				"version": null,
				"is_accepted": false,
				"is_published": false
			},
			"type": "review",
			"type_crossref": "journal-article",
			"indexed_in": [
				"crossref",
				"pubmed"
			],
			"open_access": {
				"is_oa": false,
				"oa_status": "closed",
				"oa_url": null,
				"any_repository_has_fulltext": false
			},
			"authorships": [
				{
					"author_position": "first",
					"author": {
						"id": "https://openalex.org/A5050075935",
						"display_name": "Peter Daszak",
						"orcid": "https://orcid.org/0000-0002-2046-5695"
					},
					"institutions": [
						{
							"id": "https://openalex.org/I165733156",
							"display_name": "University of Georgia",
							"ror": "https://ror.org/00te3t702",
							"country_code": "US",
							"type": "funder",
							"lineage": [
								"https://openalex.org/I165733156"
							]
						}
					],
					"countries": [
						"US"
					],
					"is_corresponding": true,
					"raw_author_name": "Peter Daszak",
					"raw_affiliation_strings": [
						"Infectious Disease and Pathology Activity, Division of Viral and Rickettsial Diseases, National Center for Infectious Diseases, Centers for Disease Control and Prevention, Atlanta, GA 30333, USA.",
						"Institute of Ecology, University of Georgia, Athens, GA 30602, USA."
					],
					"affiliations": [
						{
							"raw_affiliation_string": "Institute of Ecology, University of Georgia, Athens, GA 30602, USA.",
							"institution_ids": [
								"https://openalex.org/I165733156"
							]
						},
						{
							"raw_affiliation_string": "Infectious Disease and Pathology Activity, Division of Viral and Rickettsial Diseases, National Center for Infectious Diseases, Centers for Disease Control and Prevention, Atlanta, GA 30333, USA.",
							"institution_ids": [
								"https://openalex.org/I4210160401",
								"https://openalex.org/I1289490764"
							]
						}
					]
				},
				{
					"author_position": "middle",
					"author": {
						"id": "https://openalex.org/A5027770167",
						"display_name": "Andrew A. Cunningham",
						"orcid": "https://orcid.org/0000-0002-3543-6504"
					},
					"institutions": [
						{
							"id": "https://openalex.org/I1321067030",
							"display_name": "Zoological Society of London",
							"ror": "https://ror.org/03px4ez74",
							"country_code": "GB",
							"type": "nonprofit",
							"lineage": [
								"https://openalex.org/I1321067030"
							]
						}
					],
					"countries": [
						"GB"
					],
					"is_corresponding": false,
					"raw_author_name": "Andrew A. Cunningham",
					"raw_affiliation_strings": [
						"Institute of Zoology, Zoological Society of London, Regent's Park, London NW1 4RY, UK."
					],
					"affiliations": [
						{
							"raw_affiliation_string": "Institute of Zoology, Zoological Society of London, Regent's Park, London NW1 4RY, UK.",
							"institution_ids": [
								"https://openalex.org/I1321067030"
							]
						}
					]
				},
				{
					"author_position": "last",
					"author": {
						"id": "https://openalex.org/A5110165479",
						"display_name": "Alex D. Hyatt",
						"orcid": null
					},
					"institutions": [
						{
							"id": "https://openalex.org/I1292875679",
							"display_name": "Commonwealth Scientific and Industrial Research Organisation",
							"ror": "https://ror.org/03qn8fb07",
							"country_code": "AU",
							"type": "government",
							"lineage": [
								"https://openalex.org/I1292875679",
								"https://openalex.org/I2801453606",
								"https://openalex.org/I4387156119"
							]
						},
						{
							"id": "https://openalex.org/I1338668087",
							"display_name": "Australian Centre for Disease Preparedness",
							"ror": "https://ror.org/02aseym49",
							"country_code": "AU",
							"type": "facility",
							"lineage": [
								"https://openalex.org/I2801453606",
								"https://openalex.org/I4387156119"
							]
						}
					],
					"countries": [
						"AU"
					],
					"is_corresponding": false,
					"raw_author_name": "Alex D. Hyatt",
					"raw_affiliation_strings": [
						"Australian Animal Health Laboratory, CSIRO, Private Bag 24, Geelong, Victoria 3220, Australia."
					],
					"affiliations": [
						{
							"raw_affiliation_string": "Australian Animal Health Laboratory, CSIRO, Private Bag 24, Geelong, Victoria 3220, Australia.",
							"institution_ids": [
								"https://openalex.org/I1292875679",
								"https://openalex.org/I1338668087"
							]
						}
					]
				}
			],
			"institution_assertions": [

			],
			"countries_distinct_count": 3,
			"institutions_distinct_count": 6,
			"corresponding_author_ids": [
				"https://openalex.org/A5050075935"
			],
			"corresponding_institution_ids": [
				"https://openalex.org/I165733156",
				"https://openalex.org/I4210160401",
				"https://openalex.org/I1289490764"
			],
			"apc_list": null,
			"apc_paid": null,
			"fwci": 20.016,
			"has_fulltext": true,
			"fulltext_origin": "ngrams",
			"cited_by_count": 4120,
			"citation_normalized_percentile": {
				"value": 0.998267,
				"is_in_top_1_percent": true,
				"is_in_top_10_percent": true
			},
			"cited_by_percentile_year": {
				"min": 99,
				"max": 100
			},
			"biblio": {
				"volume": "287",
				"issue": "5452",
				"first_page": "443",
				"last_page": "449"
			},
			"is_retracted": false,
			"is_paratext": false,
			"primary_topic": {
				"id": "https://openalex.org/T12492",
				"display_name": "Zoonotic diseases and public health",
				"score": 0.9996,
				"subfield": {
					"id": "https://openalex.org/subfields/2739",
					"display_name": "Public Health, Environmental and Occupational Health"
				},
				"field": {
					"id": "https://openalex.org/fields/27",
					"display_name": "Medicine"
				},
				"domain": {
					"id": "https://openalex.org/domains/4",
					"display_name": "Health Sciences"
				}
			},
			"topics": [
				{
					"id": "https://openalex.org/T12492",
					"display_name": "Zoonotic diseases and public health",
					"score": 0.9996,
					"subfield": {
						"id": "https://openalex.org/subfields/2739",
						"display_name": "Public Health, Environmental and Occupational Health"
					},
					"field": {
						"id": "https://openalex.org/fields/27",
						"display_name": "Medicine"
					},
					"domain": {
						"id": "https://openalex.org/domains/4",
						"display_name": "Health Sciences"
					}
				}
			],
			"keywords": [
				{
					"id": "https://openalex.org/keywords/one-health",
					"display_name": "One Health",
					"score": 0.4474327
				}
			],
			"concepts": [
				{
					"id": "https://openalex.org/C29376679",
					"wikidata": "https://www.wikidata.org/wiki/Q241741",
					"display_name": "Wildlife",
					"level": 2,
					"score": 0.9107766
				}
			],
			"mesh": [
				{
					"descriptor_ui": "D000835",
					"descriptor_name": "Animals, Wild",
					"qualifier_ui": "",
					"qualifier_name": null,
					"is_major_topic": true
				},
			],
			"locations_count": 2,
			"locations": [
				{
					"is_oa": false,
					"landing_page_url": "https://doi.org/10.1126/science.287.5452.443",
					"pdf_url": null,
					"source": {
						"id": "https://openalex.org/S3880285",
						"display_name": "Science",
						"issn_l": "0036-8075",
						"issn": [
							"0036-8075",
							"1095-9203"
						],
						"is_oa": false,
						"is_in_doaj": false,
						"is_indexed_in_scopus": true,
						"is_core": true,
						"host_organization": "https://openalex.org/P4310315823",
						"host_organization_name": "American Association for the Advancement of Science",
						"host_organization_lineage": [
							"https://openalex.org/P4310315823"
						],
						"host_organization_lineage_names": [
							"American Association for the Advancement of Science"
						],
						"type": "journal"
					},
					"license": null,
					"license_id": null,
					"version": null,
					"is_accepted": false,
					"is_published": false
				},
				{
					"is_oa": false,
					"landing_page_url": "https://pubmed.ncbi.nlm.nih.gov/10642539",
					"pdf_url": null,
					"source": {
						"id": "https://openalex.org/S4306525036",
						"display_name": "PubMed",
						"issn_l": null,
						"issn": null,
						"is_oa": false,
						"is_in_doaj": false,
						"is_indexed_in_scopus": false,
						"is_core": false,
						"host_organization": "https://openalex.org/I1299303238",
						"host_organization_name": "National Institutes of Health",
						"host_organization_lineage": [
							"https://openalex.org/I1299303238"
						],
						"host_organization_lineage_names": [
							"National Institutes of Health"
						],
						"type": "repository"
					},
					"license": null,
					"license_id": null,
					"version": null,
					"is_accepted": false,
					"is_published": false
				}
			],
			"best_oa_location": null,
			"sustainable_development_goals": [
				{
					"score": 0.72,
					"display_name": "Life on land",
					"id": "https://metadata.un.org/sdg/15"
				}
			],
			"grants": [

			],
			"datasets": [

			],
			"versions": [

			],
			"referenced_works_count": 124,
			"referenced_works": [
				"https://openalex.org/W1531076819",
				"https://openalex.org/W1557724928",
				"https://openalex.org/W74730613"
			],
			"related_works": [
				"https://openalex.org/W634588095",
				"https://openalex.org/W4319429997",
				"https://openalex.org/W2605546711"
			],
			"abstract_inverted_index": {
				"Emerging": [
					0
				],

				"biodiversity.": [
					93
				]
			},
			"abstract_inverted_index_v3": null,
			"cited_by_api_url": "https://api.openalex.org/works?filter=cites:W2150891147",
			"counts_by_year": [
				{
					"year": 2025,
					"cited_by_count": 45
				},
				{
					"year": 2024,
					"cited_by_count": 159
				},
				{
					"year": 2012,
					"cited_by_count": 212
				}
			],
			"updated_date": "2025-05-27T02:52:19.182644",
			"created_date": "2016-06-24",
			"downloaded": false,
			"pdfdownloaded": false,
			"jsondownloaded": false,
			"csvmade": false,
			"htmlmade": false
		},

	"total_hits": 5,
	"cursor_mark": "IlsyNzMuMzMxNjcsIDEzMDQyOTQ0MDAwMDAsICdodHRwczovL29wZW5hbGV4Lm9yZy9XMjE0MjIxMTQxMiddIg=="
}
    """
    # json_dict = AmiJson.read_nested_dicts()
    RESULTS_JSON = "openalex-results.json"

    def __init__(self):
        pass

    @classmethod
    def read_json_create_write_html_table(cls, infile, outfile, wanted_keys):
        logger.warning("read_json_create_write_html_table not yet written")

