import glob
import logging
import os
import pprint
import re
import traceback
import unittest
from collections import defaultdict
from pathlib import Path

import lxml
import requests
from lxml import etree
from lxml.etree import XMLSyntaxError, _Element
import lxml.etree as ET

# from github import Github

# local
from amilib.ami_dict import AmiDictionary, AmiEntry, AMIDictError, \
    AmiDictValidator, NAME, TITLE, TERM, LANG_UR, VERSION, WIKIDATA_ID
from amilib.amix import AmiLib
from amilib.constants import LOCAL_CEV_OPEN_DICT_DIR
from amilib.dict_args import AmiDictArgs
from amilib.dict_args import MARKUP_FILE
from amilib.file_lib import FileLib
from amilib.util import Util
from amilib.wikimedia import WikidataSparql
from amilib.xml_lib import XmlLib
from amilib.ami_html import HtmlLib
from test.resources import Resources
from test.test_all import AmiAnyTest

# from parametrized import parametrized

# MUST use RAW content , not HTML
CEV_OPEN_RAW_DICT_URL = "https://raw.githubusercontent.com/petermr/CEVOpen/master/dictionary/"
PLANT_PART_RAW_DICT_URL = CEV_OPEN_RAW_DICT_URL + "eoPlantPart/eoplant_part.xml"
COMPOUND_RAW_DICT_URL = CEV_OPEN_RAW_DICT_URL + "eoCompound/plant_compound.xml"
ANALYSIS_METHOD_RAW_DICT_URL = CEV_OPEN_RAW_DICT_URL + "eoAnalysisMethod/eoAnalysisMethod.xml"
TEST_DIR = Path(Path(__file__).parent.parent, "test")
TEST_RESOURCE_DIR = Path(TEST_DIR, "resources")
DICTFILE1 = "dictfile1"
ROOT = "root"
ONE_ENTRY_PATH = "one_entry_file"
ONE_ENTRY_DICT = "one_entry_dict"
MINI_PLANT_PART = "mini_plant_part"
MINI_MENTHA = "mini_mentha"
ETHNOBOT_DICT = "VC_EthnobotanicalUse"
DUPLICATE_ENTRIES = "test_duplicate_entries"

RESOURCES_DIR = "resources"
# AMIDICTS = Path(Path(__file__).parent.parent, "py4ami/resources/amidicts")  # relative to dictribution base
AMIDICTS = Path(Path(__file__).parent.parent, f"{RESOURCES_DIR}/amidicts")  # relative to dictribution base
AMIDICTS = Path(Path(__file__).parent.parent, f"{TEST_RESOURCE_DIR}/amidicts")  # relative to dictribution base

STARTING_VERSION = "0.0.1"

HOME = os.path.expanduser("~")
IPCC_DICT_ROOT = Path(HOME, "projects/semanticClimate/ipcc/ar6/wg3")  # needs resetting if tests are to be run

logger = Util.get_logger(__name__, level=logging.DEBUG)

# ===== helpers =====
def _create_amidict_with_foo_bar_entries():
    amidict = AmiDictionary.create_minimal_dictionary()
    entry_foo = amidict.create_and_add_entry_with_term("foo")
    entry_bar = amidict.create_and_add_entry_with_term("bar")
    return amidict

class AmiIndexMisc:
    """
    temporary class for components that will move to more formal library
    """
    from collections import Counter
    @classmethod
    def read_counter(cls, url: str) -> Counter:
        """
        Read a serialized Counter object using ast.literal_eval
        :param url:
        """
        import requests
        import ast
        from collections import Counter

        # Fetch the serialized data from the URL
        response = requests.get(url)

        # Ensure the request was successful
        if response.status_code == 200:
            try:
                # Get the string representing the Counter (e.g. Counter({'key': value, ...}))
                counter_str = response.text.strip()

                # Remove the 'Counter(' and ')' parts to get the dictionary-like string
                if counter_str.startswith('Counter(') and counter_str.endswith(')'):
                    counter_dict_str = counter_str[8:-1]  # Removes 'Counter(' and ')'

                    # Safely evaluate the dictionary string to get a dict
                    counter_dict = ast.literal_eval(counter_dict_str)

                    # Ensure that the deserialized object is a dictionary
                    if isinstance(counter_dict, dict):
                        counter = Counter(counter_dict)
                        print(f"Deserialized Counter: {len(counter)} {counter[:20]}")
                        return counter
                    else:
                        print("The deserialized object is not a dictionary.")
                else:
                    print("The string does not represent a valid Counter.")
            except (ValueError, SyntaxError) as e:
                print(f"Error during deserialization: {e}")
        else:
            print(f"Failed to fetch the data. Status code: {response.status_code}")

    @classmethod
    def create_html_index_of_pages(cls, counter, max_page, min_count, pages_url, start, stopwords):
        page_dict = defaultdict(list)
        page_urls = cls.get_climate_academy_page_urls(pages_url, max_page, start=start)
        for page_url in page_urls:
            url_bits = page_url.split("/")
            page_no = url_bits[-1][len('page_'):-4]
            # download url
            response = requests.get(page_url)
            # text of url
            data = response.text
            # split by whitespace (spaces and newlines)
            words = data.split()
            for word in words:
                if word in stopwords:
                    continue
                if word in counter:
                    count = int(counter[word])
                    if count <= min_count:
                        continue
                    page_dict[word].append(page_no)
        html = HtmlLib.create_html_with_empty_head_body()
        body = HtmlLib.get_body(html)
        cls.add_and_sort_words_and_hyperlinks_to_pages(body, page_dict, pages_url)
        return html

    @classmethod
    def add_and_sort_words_and_hyperlinks_to_pages(cls, body, page_dict, pages_url):
        ul = ET.SubElement(body, "ul")
        words = page_dict.keys()
        for word in sorted(words, key=str.casefold):
            li = ET.SubElement(ul, "li")
            span = ET.SubElement(li, "span")
            span.text = f"{word}: "
            page_list = sorted(set(page_dict[word]), key=int)
            page_dict[word] = page_list
            for pg in page_list:
                # generate URL or text page
                pg_url = f"{pages_url}/page_{pg}.txt"
                a = ET.SubElement(li, "a")
                a.attrib["href"] = pg_url
                a.text = f"{pg},"

    @classmethod
    def get_climate_academy_page_urls(cls, pages_url, max_page, start=1):
        """
        gets all pages from semanticClimate ClimateAcademy book
        :param max_page: maximum pages to download
        :param pages_url: URL of pages
        :param start: start page
        :return: list of urls in numeric order
        """
        page_nos = range(1, max_page + 1)  # because count from 1
        page_urls = []
        for page_no in page_nos:
            if page_no < 10:
                page_no = f"0{page_no}" # leading zero if < 10!
            page_url = f"{pages_url}/page_{page_no}.txt"
            page_urls.append(page_url)
        return page_urls

class AmiDictionaryTest(AmiAnyTest):
    """These are tests for developing CODE for dictionary creation and validation

    Code for VALIDATION of dictionaries should probably be bundled with the dictionaries themselves

    """

    logging.info(f"loading {__file__}")

    HTML_WITH_IDS = "html_with_ids"

    DICTFILE1 = "dictfile1"
    ROOT = "root"
    ONE_ENTRY_PATH = "one_entry_file"
    ONE_ENTRY_DICT = "one_entry_dict"
    MINI_PLANT_PART = "mini_plant_part"
    MINI_MENTHA = "mini_mentha"
    ETHNOBOT_DICT = "VC_EthnobotanicalUse"
    DUPLICATE_ENTRIES = "test_duplicate_entries"

    # AMIDICTS = Path(Path(__file__).parent.parent, "py4ami/resources/amidicts")  # relative to dictribution base
    # AMIDICTS = Path(Path(__file__).parent.parent, f"{AMI_TOP}/resources/amidicts")  # relative to dictribution base

    STARTING_VERSION = "0.0.1"

    ADMIN = True and AmiAnyTest.ADMIN
    CMD = True and AmiAnyTest.CMD
    LONG = True and AmiAnyTest.LONG
    NET = True and AmiAnyTest.NET
    NYI = True and AmiAnyTest.NYI
    USER = True and AmiAnyTest.USER
    VERYLONG = True and AmiAnyTest.VERYLONG

    CEV_EXISTS = Path(LOCAL_CEV_OPEN_DICT_DIR).exists()

    def create_file_dict(self):
        """Variables created afresh for every test"""
        dictfile1 = Path(AMIDICTS, "dict1.xml")
        # dictfile1 = Path(Path(__file__).parent.parent, "py4ami/resources/amidicts/dict1.xml")
        one_entry_path = Path(Path(__file__).parent.parent, f"{AMIDICTS}/dict_one_entry.xml")
        root = etree.parse(str(dictfile1)).getroot()
        assert dictfile1.exists(), "{dictfile1} exists"
        one_entry_path = Path(AMIDICTS, "dict_one_entry.xml")
        one_entry_dict_new = AmiDictionary.create_from_xml_file(one_entry_path)
        assert one_entry_dict_new is not None
        mini_plant_part_path = Path(AMIDICTS, "mini_plant_part.xml")

        # BUG: this should be available through pytest
        setup_dict = {
            DICTFILE1: dictfile1,  # type path
            ROOT: root,
            ONE_ENTRY_PATH: one_entry_path,
            ONE_ENTRY_DICT: one_entry_dict_new,
            MINI_PLANT_PART: mini_plant_part_path,
            MINI_MENTHA: Path(AMIDICTS, "mentha_tps.xml"),
            ETHNOBOT_DICT: Path(AMIDICTS, ETHNOBOT_DICT + ".xml"),
            DUPLICATE_ENTRIES: Path(AMIDICTS, DUPLICATE_ENTRIES + ".xml"),
        }
        logger.info(f"setup_dict {setup_dict}")
        return setup_dict


    def test_dictionary_element(self):
        """
        reads xml string into dictiomary and tests that has "dictionary" root element
        and no child entries
        """
        dict_str = """
        <dictionary title='foo'>
        </dictionary>
        """
        ami_dict = AmiDictionary.create_dictionary_from_xml_string(dict_str)
        assert ami_dict is not None
        assert ami_dict.root.tag == "dictionary"
        assert ami_dict.has_valid_root_tag()
        assert ami_dict.get_entry_count() == 0

    def test_title_from_url_stem(self):
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.url = "https://some.where/foo/bar.xml"
        assert amidict.root.attrib[TITLE] == "minimal"  # needs fixing

    def test_title_from_file_stem(self):
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.file = "/user/me/foo.xml"
        assert amidict.root.attrib[TITLE] == "minimal"

    def test_dict_has_root_dictionary(self):
        """
        Tests that the dictionary has <dictionary> root element
        """
        setup_dict = self.create_file_dict()
        root = setup_dict[ROOT]
        assert root.tag == AmiDictionary.TAG

    def test_dict_contains_xml_element(self):
        root = etree.parse(str(self.create_file_dict()[DICTFILE1]))
        assert root is not None

    def test_can_read_dictionary_from_url_as_xml(self):
        """
        Checks that a dictionary can be read from a URL into XML
        Reads PLANT_PART_RAW_DICT_URL and validates that has about 728 entries
        """

        url = PLANT_PART_RAW_DICT_URL
        tree = XmlLib.parse_url_to_tree(url)
        descendants = tree.getroot().xpath('.//*')
        assert 730 >= len(descendants) >= 720

    def test_dictionary_has_xml_declaration_with_encoding(self):
        """Checks dictionary has encoding of 'UTF-8' and XML Version 1.0
        USEFUL 2022-07"""
        dicts = [ETHNOBOT_DICT, DICTFILE1, ]
        for dikt in dicts:
            logger.debug(f"...{dikt}")
            root = etree.parse(str(self.create_file_dict()[dikt]))
            dictionary = AmiDictionary.create_from_xml_object(root)
            validator = AmiDictValidator(dictionary)
            error_list = validator.get_xml_declaration_error_list()
            assert not error_list

    # @pytest.mark.url
    def test_validate_url_dict(self):
        """
        tests that historic dictionaries read into validator
        TODO skip this
        """
        urllist = [
            PLANT_PART_RAW_DICT_URL,
            ANALYSIS_METHOD_RAW_DICT_URL,
            COMPOUND_RAW_DICT_URL
        ]
        for url in urllist:
            logger.debug(f"url: {url}")
            # tree = XmlLib.parse_url_to_tree(url)
            # dictionary = AmiDictionary.create_from_xml_object(tree)
            dictionary = AmiDictionary.create_dictionary_from_url(url)
            validator = AmiDictValidator(dictionary)
            # validator.validate_title()
            error_list = validator.get_error_list()
            logger.warning(f"error_list {error_list}")

    # AmiDictionary

    def test_can_create_ami_dict_from_file(self):
        """read an existing XML AmiDictionary"""
        setup_dict = self.create_file_dict()
        one_entry_path = setup_dict[ONE_ENTRY_PATH]
        amidict = AmiDictionary.create_from_xml_file(one_entry_path)
        assert amidict is not None

    # @pytest.mark.simple
    def test_dictionary_is_an_ami_dictionary(self):
        """
        test dictionary with one entry
        """
        setup_dict = self.create_file_dict()
        amidict = setup_dict[ONE_ENTRY_DICT]
        assert type(amidict) is AmiDictionary

    def test_dictionary_get_entries(self):
        """
        unit test
        """
        setup_dict = self.create_file_dict()
        amidict = setup_dict[ONE_ENTRY_DICT]
        entries = amidict.get_lxml_entries()
        assert entries is not None

    # @pytest.mark.simple
    def test_dictionary_contains_one_entry(self):
        """
        unit test
        """
        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert amidict.get_entry_count() == 1, f"dict should have 1 entry, found  {amidict.get_entry_count()}"

    def test_get_first_entry(self):
        """
        unit test
        """
        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert amidict.get_first_ami_entry() is not None

    def test_get_attribute_names(self):
        """
        unit test
        """

        first_entry = self.create_file_dict()[ONE_ENTRY_DICT].get_first_ami_entry()
        assert type(first_entry) is AmiEntry
        attrib_names = {name for name in first_entry.element.attrib}
        assert attrib_names is not None

    def test_get_term_of_first_entry(self):
        """
        tests that we can retrieve the `term` value from an element

        """
        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert amidict.get_first_entry().attrib[TERM] == "Douglas Adams"

    def test_get_name_of_first_entry(self):
        """
        unit test
        """

        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert amidict.get_first_entry().attrib[NAME] == "Douglas Adams"

    def test_get_wikidata_of_first_entry(self):
        """
        unit test
        """

        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert amidict.get_first_entry().attrib[WIKIDATA_ID] == "Q42"

    def test_get_synonym_count(self):
        """
        unit test
        """
        amidict = AmiDictionaryTest().create_file_dict()[ONE_ENTRY_DICT]
        assert type(amidict) is AmiDictionary
        assert len(amidict.get_first_ami_entry().get_synonyms()) == 2

    def test_get_synonym_by_language(self):
        """
        unit test
        """

        amidict = self.create_file_dict()[ONE_ENTRY_DICT]
        assert type(amidict) is AmiDictionary
        elem = amidict.get_first_ami_entry().get_synonym_by_language(LANG_UR).element
        assert "ڈگلس ایڈمس" == ''.join(elem.itertext())

    def test_add_entry_with_term_to_zero_entry_dict(self):
        """
        Creates minimal dictionary with no entries and adds one entry
        tests its components
        """
        amidict = AmiDictionary.create_minimal_dictionary()
        print()
        logger.info(f"amidict {amidict.get_entry_count()}")
        entry = amidict.create_and_add_entry_with_term("foo")
        assert etree.tostring(entry) == b'<entry term="foo"/>'
        assert etree.tostring(
            amidict.root) == b'<dictionary title="minimal" version="0.0.1"><entry term="foo"/></dictionary>'
        assert amidict.get_entry_count() == 1

    def test_add_two_entry_with_term_to_zero_entry_dict(self):
        amidict = AmiDictionary.create_minimal_dictionary()
        entry_foo = amidict.add_entry_element("foo")
        entry_bar = amidict.add_entry_element("bar")
        assert etree.tostring(entry_bar) == b'<entry name="bar" term="bar"/>'
        assert etree.tostring(
            amidict.root) == b'<dictionary title="minimal" version="0.0.1"><entry name="foo" term="foo"/><entry name="bar" term="bar"/></dictionary>'
        assert amidict.get_entry_count() == 2

    def test_add_list_of_entries_from_list_of_string(self):
        """
        from a list of strings creates a list of entries and adds to existing dictionary
        (the entries only have term/name fields)
        """
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        term_count = len(terms)
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.add_entries_from_words(terms)
        assert amidict.get_entry_count() == term_count

    def test_find_entry_after_add_list_of_entries_from_list_of_string(self):
        """
        creates entries from strings and tests that they can be accessed by term
        """
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.add_entries_from_words(terms)
        entry_bar = amidict.get_lxml_entry("bar")
        assert entry_bar is not None

    def test_fail_on_missing_entry_after_add_list_of_entries_from_list_of_string(self):
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.add_entries_from_words(terms)
        entry_zilch = amidict.get_lxml_entry("zilch")
        assert entry_zilch is None, f"missing entry returns None"

    def test_add_second_list_of_entries_from_list_of_string(self):
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.add_entries_from_words(terms)
        terms1 = ["wibble", "wobble"]
        amidict.add_entries_from_words(terms1)
        assert amidict.get_entry_count() == len(terms) + len(terms1)

    def test_add_list_of_entries_from_list_of_string_with_duplicates_and_replace(self):
        """
        creates dictionary from list of words and force replacement of duplicates
        """
        terms = ["foo", "bar", "plugh", "xyzzy", "bar"]
        amidict = AmiDictionary.create_minimal_dictionary()
        amidict.add_entries_from_words(terms, duplicates="replace")
        assert amidict.get_entry_count() == 4, f"'bar' should be present"

    def test_add_list_of_entries_from_list_of_string_with_duplicates_and_no_replace(self):
        """
        add list of terms which contains duplicate and raise error
        """
        terms = ["foo", "bar", "plugh", "xyzzy", "bar"]
        amidict = AmiDictionary.create_minimal_dictionary()
        try:
            amidict.add_entries_from_words(terms, duplicates="error")
            assert False, f"AMIDict duplicate error (bar) should have been thrown"
        except AMIDictError:
            assert True, "error should have been throwm"
        assert amidict.get_entry_count() == 4, f"'bar' should be present"

    def test_add_then_remove_entry_and_replace(self):
        """create new entry , then delete, then re-add"""
        amidict, _ = AmiDictionary.create_dictionary_from_words(["foo", "bar", "plugh", "xyzzy"])
        assert amidict.get_entry_count() == 4
        amidict.delete_entry_by_term("bar")
        assert amidict.get_entry_count() == 3, f"entry 'bar' should have been removed"
        amidict.create_and_add_entry_with_term("bar")
        assert amidict.get_entry_count() == 4, f"entry 'bar' should have been re-added"

    # find entries
    def test_find_entry_by_term(self):
        """searches for entry by value of term"""
        amidict = _create_amidict_with_foo_bar_entries()
        entry = amidict.get_lxml_entry("foo")
        assert entry is not None
        assert entry.attrib[TERM] == "foo", f"should retrieve entry with term 'foo'"

    def test_find_entry_by_term_bar(self):
        amidict = _create_amidict_with_foo_bar_entries()
        entry = amidict.get_lxml_entry("bar")
        assert entry is not None

    def test_find_entry_by_term_zilch(self):
        amidict = _create_amidict_with_foo_bar_entries()
        entry = amidict.get_lxml_entry("zilch")
        assert entry is None

    def test_delete_entry_by_term_foo(self):
        amidict = _create_amidict_with_foo_bar_entries()
        logger.debug(f"amidict0 {lxml.etree.tostring(amidict.root)}")
        amidict.delete_entry_by_term("foo")
        logger.debug(f"amidict1 {lxml.etree.tostring(amidict.root)}")
        assert amidict.get_entry_count() == 1

    def test_delete_entry_by_term_foo_and_re_add(self):
        amidict = _create_amidict_with_foo_bar_entries()
        amidict.delete_entry_by_term("foo")
        amidict.create_and_add_entry_with_term("foo")
        assert amidict.get_entry_count() == 2

    def test_create_and_add_entry_with_term(self):
        term = "foo"
        amidict = AmiDictionary.create_minimal_dictionary()
        assert amidict.get_entry_count() == 0
        amidict.create_and_add_entry_with_term(term)
        assert amidict.get_entry_count() == 1
        entry = amidict.get_ami_entry(term)
        assert type(entry) is AmiEntry
        assert term == entry.get_term()

    def test_create_and_overwrite_entry_with_duplicate_term(self):
        term = "foo"
        amidict = AmiDictionary.create_minimal_dictionary()
        assert amidict.get_entry_count() == 0
        entry = amidict.create_and_add_entry_with_term(term)
        logger.debug(f"entry: {type(entry)}")
        assert isinstance(entry, _Element)
        AmiEntry.add_name(entry, "foofoo")
        amidict.create_and_add_entry_with_term(term, replace=True)
        assert amidict.get_entry_count() == 1
        entry = amidict.get_lxml_entry(term)
        assert type(entry) is _Element

        assert term == entry.attrib[TERM]
        assert NAME not in entry.attrib

    def test_create_and_fail_on_add_entry_with_duplicate_term(self):
        term = "foo"
        amidict = AmiDictionary.create_minimal_dictionary()
        entry = amidict.create_and_add_entry_with_term(term)
        try:
            amidict.create_and_add_entry_with_term(term, replace=False)
            assert False, f"should fail with duplicate entry"
        except AMIDictError as e:
            assert True, "should raise duplicate error"

    def test_create_and_overwrite_duplicate_term(self):
        """
        setting term which already exists in dictionary, and replace with new term
        """
        term = "foo"
        amidict = AmiDictionary.create_minimal_dictionary()
        ami_entry = AmiEntry.create_from_element(amidict.create_and_add_entry_with_term(term))
        assert ami_entry.get_name() is None
        ami_entry.set_name("bar")
        assert ami_entry.get_name() == "bar"
        try:
            amidict.create_and_add_entry_with_term(term, replace=True)
            assert True, f"should overwrite duplicate entry"
        except AMIDictError as e:
            assert True, "should not raise duplicate error"

    # dictionary tests
    def test_minimal_dictionary(self):
        amidict = AmiDictionary.create_minimal_dictionary()
        assert amidict.get_version() is not None
        # amidict.check_validity()
        amidict.remove_attribute(VERSION)
        if amidict.get_version() is not None:
            raise AMIDictError("should have removed version")

        try:
            amidict.check_validity()
            raise AMIDictError("should fail is_valid()")
        except Exception as e:
            logging.info(f"failed test {e}")

    def test_get_duplicate_entries(self):
        """Dictionary has two entries for 'apical' but only one for 'cone'"""
        dup_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[DUPLICATE_ENTRIES])
        entries = dup_dict.get_lxml_entries()
        assert len(entries) == 4, "one duplicate term omitted"
        entries = dup_dict.find_entries_with_term("apical")
        assert entries is not None and len(entries) == 1
        entries = dup_dict.find_entries_with_term("zilch")
        assert entries is not None and len(entries) == 0

    def test_get_terms_from_valid_dictionary(self):
        """ETHNOBOT has no multiple entries'"""
        ethno_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        terms = ethno_dict.get_terms()
        assert terms is not None
        assert len(terms) == 8
        assert terms == ['anti-fumitory', 'adaptogen', 'homeopathy variable agent', 'ethnomedicinal agent',
                         'phytochemical agent', 'phytomedical agent', 'plant-extracted agent', 'lung-tonifying agent']

    def test_get_terms_from_invalid_dictionary(self):
        """DUPLICATE_ENTRIES has two entries for 'apical' and some missing terms"""
        dup_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[DUPLICATE_ENTRIES])
        terms = dup_dict.get_terms()
        assert terms == ['apical', 'flowering top', 'cone', 'pistil']

    # review dictionaries
    def test_mini_plant_part_is_valid(self):
        # pp_dict = AmiDictionary(setup_amidict[MINI_PLANT_PART])
        pp_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[MINI_PLANT_PART])
        if pp_dict is None:
            raise AMIDictError(f"test_dictionary_should_have_desc cannot read dictionary {pp_dict}")
        pp_dict.check_validity()

    def test_mini_mentha_tps_dict_is_valid(self):
        mentha_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[MINI_MENTHA])
        if mentha_dict is None:
            raise AMIDictError("cannot find/read mentha_dict")
        mentha_dict.check_validity()

    def test_ethnobot_dict_has_version(self):
        ethnobot_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        version = ethnobot_dict.get_version()
        assert version is not None
        assert AmiDictionary.is_valid_version_string(version)
        # assert ethnobot_dict.get_version() == "0.0.1"

    def test_ethnobot_dict_is_valid(self):
        logger.info(f" validating {ETHNOBOT_DICT}")
        ethnobot_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        ethnobot_dict.check_validity()
        # assert ethnobot_dict.get_version() == "0.0.1"

    def test_ethnobot_dict_has_8_entries(self):
        ethnobot_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        entries = ethnobot_dict.get_lxml_entries()
        assert len(entries) == 8

    def test_ethnobot_dict_entry_0_is_valid(self):
        ethnobot_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        entry0 = ethnobot_dict.get_lxml_entries()[0]
        AmiEntry.create_from_element(entry0).check_validity()

    def test_all_ethnobot_dict_entries_are_valid(self):
        ethnobot_dict = AmiDictionary.create_from_xml_file(self.create_file_dict()[ETHNOBOT_DICT])
        for entry in ethnobot_dict.get_lxml_entries():
            AmiEntry.create_from_element(entry).check_validity()

    # integrations

    def test_find_missing_wikidata_ids(self):
        ami_dict = AmiDictionary.create_from_xml_file(Resources.TEST_IPCC_CHAP02_ABB_DICT)
        lxml_entries = ami_dict.get_lxml_entries_with_missing_wikidata_ids()
        # missing_wikidata_ids = AmiEntry.get_wikidata_ids_for_entries(_entries)
        missing_wikidata_terms = AmiEntry.get_terms_for_lxml_entries(lxml_entries)
        assert missing_wikidata_terms == [

            'HFCs',
            'HCFCs',
            'CRF',
            'WMO',
            'NGHGI',
            'GWP',
            'FFI',
            'PBEs',
            'TCBA',
            'HCEs',
            'EBEs',
            'IBE',
            'RSD',
            'HDI',
            #            'CSP',
            'BECCS',
            'IAMs',
            # 'CDR',
            'ECR',
            'ETSs',
            #            'EVs',
            'ODSs',
            'HCS'
        ]


    @classmethod
    def debug_dict(cls, dict_path):
        logger.info(f"======={dict_path}=======")
        ami_dict = AmiDictionary.create_from_xml_file(dict_path)
        if ami_dict:
            lookup = ami_dict.lookup_missing_wikidata_ids()
            pprint.PrettyPrinter(indent=4).pprint(lookup.hits_dict)
        else:
            print(f"****Cannot find valid dict {dict_path}****")
            logging.error(f"Cannot find valid dict {dict_path}")

    # TODO set changes over time
    def test_get_property_ids(self):
        """gets properties af a dictionary entry"""
        words = ["limonene"]
        dictionary, _ = AmiDictionary.create_dictionary_from_words(
            words, "test", "created from words",wikilangs=["en", "de"])
        dictionary.add_wikidata_from_terms()
        pprint.pprint(lxml.etree.tostring(dictionary.root).decode("UTF-8"))
        assert len(dictionary.entries) == 1
        wikidata_page = dictionary.create_wikidata_page(dictionary.entries[0])
        property_ids = wikidata_page.get_property_ids()
        assert len(property_ids) >= 60
        id_set = set(property_ids)
        test_set = {'P31', 'P279', 'P361', 'P2067', 'P274', 'P233', 'P2054', 'P2101', 'P2128', 'P2199'}
        assert test_set.issubset(id_set)

    def test_merge_dicts_ipcc_same_chap(self):
        """test merge dictionaries from IPCC (heavy commonality)"""

        abb2_dict = AmiDictionary.create_from_xml_file(Resources.TEST_IPCC_CHAP02_ABB_DICT)
        abb2_set = abb2_dict.get_or_create_term_set()
        assert abb2_set == {
            'BECCS',
            #            'CBEs',
            #            'CDR',
            'CRF',
            #            'CSP',
            'EBEs', 'ECR',
            # 'EET',
            'ETSs',
            'EU ETS',
            #            'EVs',
            #            'F-gases', 'FAQs',
            'FFI',
            # 'GDP', 'GHG',
            'GTP',
            'GWP',
            #            'GWP100',
            'HCEs', 'HCFCs',
            'HCS', 'HDI', 'HFCs', 'IAMs',
            'IBE', 'LULUCF', 'NGHGI', 'ODSs',
            'PBEs',
            # 'PFCs',
            'RGGI', 'RSD',
            'TCBA',
            # 'UNFCCC',
            'WMO'
        }, f"abb2 set {abb2_set}"

        man2_dict = AmiDictionary.create_from_xml_file(
            Path(Resources.TEST_IPCC_CHAP02_DICT, "ip_3_2_emissions_man.xml"))
        man2_set = man2_dict.get_or_create_term_set()
        assert man2_set == {
            'CAIT', 'CEDS', 'CGTP', 'CO2-equivalent emission',
            'CRF', 'EDGAR', 'FAOSTAT', 'FFI', 'FOLU', 'Final Energy Demand', 'GTP', 'GWP',
            'GWP100', 'GtCO2eq', 'LULUCF', 'NMVOC',
            'PRIMAP', 'Paris Agreement', 'Primary Energy', 'Primary Energy Conversion',
            'SLCF', 'SRES', 'SSP', 'UNFCCC', 'WMO',
            'atmospheric lifetime', 'baseline scenario',
            'carbon budget', 'carbon pricing',
            'cumulative CO2 emissions', 'demand side solutions',
            'emission inventory', 'emission sectors',
            'emissions factor', 'emissions trajectory',
            'fluorinated gas', 'social discount rate',
            'top down atmospheric measurement'
        }, f"man2 set {man2_set}"

        # phrases
        phr2_dict = AmiDictionary.create_from_xml_file(
            Path(Resources.TEST_IPCC_CHAP02_DICT, "ip_3_2_emissions_phr.xml"))
        phr2_set = phr2_dict.get_or_create_term_set()
        assert phr2_set == {
            'BECCS', 'CBEs', 'CDR', 'CRF', 'CSP', 'EBEs', 'ECR', 'EET',
            'ETSs', 'EU ETS', 'EVs', 'F-gases',
            'FAQs', 'FFI', 'GDP', 'GHG', 'GTP', 'GWP', 'GWP-100', 'GWP100',
            'HCEs', 'HCFCs', 'HCS', 'HDI', 'HFCs', 'IAMs', 'IBE', 'LULUCF',
            'NGHGI', 'ODSs', 'PBEs', 'PFCs', 'RGGI', 'RSD', 'TCBA', 'UNFCCC', 'WMO'
        }

        # terms common to abbrev and manual
        abb_man_set = abb2_set.intersection(man2_set)
        assert len(abb_man_set) == 6, f"man2 set {len(abb_man_set)}"
        assert abb_man_set == {
            'GTP', 'FFI',
            #            'GWP', 'UNFCCC',
            'WMO', 'GWP', 'CRF', 'LULUCF'}

    def test_entries_have_ids(self):
        dict_file = Path(Resources.TEST_IPCC_DICT_DIR, "raw_linked_dict.xml")
        assert dict_file.exists(), f"file {dict_file} should exist"
        linked_dict = AmiDictionary.create_from_xml_file(dict_file)
        assert linked_dict is not None
        assert linked_dict.get_entry_count() == 17, f"found {linked_dict.get_entry_count()} entries"
        # assert linked_dict.get_lxml_entries_with_ids() == 2

    def test_entry_id(self):
        """
        Tests whether entries have ids, and (later) creates default values
        """
        dict_dir = Path(Resources.TEST_RESOURCES_DIR, "eoCompound")
        assert dict_dir.exists(), f"{dict_dir} should exist"
        ambig_dict_file = Path(dict_dir, "disambig.xml")
        assert ambig_dict_file.exists(), f"{ambig_dict_file} should exist"
        disambig_dict = AmiDictionary.create_from_xml_file(ambig_dict_file)
        assert disambig_dict is not None

        assert disambig_dict.get_entry_count() == 9, f"dictionary should have 9 entries"
        assert disambig_dict.get_entry_ids() == ['____alloaromadendrene']

    def test_dict_commands_COMMAND_OK(self):
        """
        prints help
        """
        amilib = AmiLib()
        amilib.run_command(["--help"])

        amilib.run_command(["DICT", "--help"])

    def test_search_with_dictionary_and_make_links_IMPORTANT(self):
        """
        uses a simple dictionary to search WG chapter (wg2/ch03) *html_with_ids)
        
        OPTIMIZATION: Limited to 100 paragraphs instead of 1000+ for faster testing while maintaining search functionality

        Returns
        -------

        """

        chapter_file = Path(Resources.TEST_RESOURCES_DIR, "ipcc", "wg3", "Chapter03", f"{self.HTML_WITH_IDS}.html")
        # OPTIMIZATION: Limit to first 100 paragraphs instead of all (~1000+)
        paras = HtmlLib._extract_paras_with_ids(chapter_file, count=100)
        assert len(paras) <= 100, f"Expected at most 100 paragraphs, got {len(paras)}"
        xml_dict_path = Path(Resources.TEST_RESOURCES_DIR, "dictionary", "climate", "climate_words.xml")
        dictionary = AmiDictionary.create_from_xml_file(xml_dict_path)
        assert dictionary is not None
        phrases = dictionary.get_terms()
        html_path = Path(Resources.TEST_RESOURCES_DIR, "dictionary", "climate", "climate_words.html")
        # OPTIMIZATION: Skip HTML generation if file already exists
        if not html_path.exists():
            dictionary.create_html_write_to_file(html_path, debug=False)
        dictionary.location = html_path
        assert len(phrases) == 11  # Keep original - this is from the dictionary file
        para_phrase_dict = HtmlLib.search_phrases_in_paragraphs(paras, phrases, markup=html_path)
        # Verify we found at least one match (one lookup)
        assert len(para_phrase_dict) > 0, "Should find at least one phrase match"
        chapter_elem = paras[0].xpath("/html")[0]
        chapter_outpath = Path(Resources.TEMP_DIR, "ipcc", "Chapter03", "marked_up.html")
        HtmlLib.write_html_file(chapter_elem, chapter_outpath, debug=False)

    def test_search_with_dictionary_and_make_links_WORKFLOW_LONG(self):
        """
        uses a simple dictionary to search WG chapter (wg2/ch03) *html_with_ids)

        Returns
        -------

        """
        stem = "carbon_cycle"
        MAXPARAS = 100 # to limit test time
        NTERMS = 5
        words_path = Path(Resources.TEST_RESOURCES_DIR, "wordlists", f"{stem}_edited_small.txt")
        assert words_path.exists()

        chapter_file = Path(Resources.TEST_RESOURCES_DIR, "ar6", "wg1", "Chapter05", f"{self.HTML_WITH_IDS}.html")
        assert chapter_file.exists()

        paras = HtmlLib._extract_paras_with_ids(chapter_file, count=1724)

        dictionary, outpath = AmiDictionary.create_dictionary_from_wordfile(words_path)
        assert dictionary is not None
        assert len(dictionary.get_terms()) == NTERMS

        xml_dict_path = Path(Resources.TEMP_DIR, "dictionary", "climate", f"{stem}.xml")
        dictionary.create_html_write_to_file(xml_dict_path, debug=True)
        assert xml_dict_path.exists()

        html_dict_path = Path(Resources.TEMP_DIR, "dictionary", "climate", f"{stem}.html")
        dictionary.create_html_write_to_file(html_dict_path, debug=True)
        assert html_dict_path.exists()

        phrases = dictionary.get_terms()
        dictionary.location = html_dict_path
        assert len(phrases) == NTERMS
        para_phrase_dict = HtmlLib.search_phrases_in_paragraphs(paras[:MAXPARAS], phrases, markup=html_dict_path)

        # write marked_up html
        chapter_elem = paras[0].xpath("/html")[0]
        chapter_outpath = Path(Resources.TEMP_DIR, "ipcc", "wg1", "Chapter05", "marked_up.html")
        HtmlLib.write_html_file(chapter_elem, chapter_outpath, debug=True)
        assert  chapter_outpath.exists()

    def test_search_with_dictionary_and_make_links_code(self):
        """
        uses a simple dictionary to search WG chapter (wg2/ch03) *html_with_ids) and add hyperlinks
        -------

        """
        from amilib.search_args import AmiSearch

        stem = "carbon_cycle"
        chapter_file = Path(Resources.TEST_RESOURCES_DIR, "ar6", "wg1", "Chapter05", f"{self.HTML_WITH_IDS}.html")
        chapter_outpath = Path(Resources.TEMP_DIR, "ipcc", "wg1", "Chapter05", "marked_up.html")
        FileLib.delete_file(chapter_outpath)
        html_dict_path = Path(Resources.TEMP_DIR, "dictionary", "climate", f"{stem}.html")

        AmiSearch.markup_html_file_with_words_or_dictionary(
            chapter_file, chapter_outpath, html_dict_path=html_dict_path)
        assert chapter_outpath.exists()

    def test_search_with_dictionary_and_make_links_COMMANDLINE(self):
        """
        same logic and files as test_search_with_dictionary_and_make_links_CODE. Check that that runs
        This fails the first time it's run and works the second. It's not
        creating the output file.
        NEEDS DEBUGGING
        """

        stem = "carbon_cycle"
        chapter_file = str(Path(Resources.TEST_RESOURCES_DIR, "ar6", "wg1", "Chapter05", f"{self.HTML_WITH_IDS}.html"))
        chapter_outpath = str(Path(Resources.TEMP_DIR, "ipcc", "wg1", "Chapter05", "marked_up.html"))
        FileLib.delete_file(chapter_outpath)
        assert not Path(chapter_outpath).exists(), f"no output file {chapter_outpath}"
        html_dict_path = Path(Resources.TEST_RESOURCES_DIR, "dictionary", "climate", f"{stem}.html")
        assert html_dict_path.exists(), f"dictionary path {html_dict_path}"

        # commandline
        args = [
            "DICT",
            "--inpath", chapter_file,
            "--outpath", chapter_outpath,
            "--dict", str(html_dict_path),
            "--operation", MARKUP_FILE,
        ]
        logger.info(f"cmd> {' '.join(args)}")
        AmiLib().run_command(args)
        assert Path(chapter_outpath).exists(), f"file {chapter_outpath}"

    def test_args_help(self):
        args = [
            "DICT",
            "--help"
        ]
        AmiLib().run_command(args)

class AmiEntryTest(AmiAnyTest):
    """
    test functionality of AmiEntry
    """

    def dummy(self):
        pass

class ValidateTest(AmiAnyTest):
    """
    test validity of AmiEntry and AmiDictionary
    """

    def test_read_wellformed_dictionary(self):
        """test can create from XML string
        includes well-formed and non-well-formed XML
        """
        dict_str = """
        <dictionary title='foo'>
        </dictionary>
        """
        ami_dict = AmiDictionary.create_dictionary_from_xml_string(dict_str)
        assert ami_dict is not None

        assert ami_dict.root.tag == "dictionary"

        dict_str = """
        <diktionary title='foo'>
        </dictionary>
        """
        try:
            ami_dict = AmiDictionary.create_dictionary_from_xml_string(dict_str)
        except XMLSyntaxError as e:
            logger.warning(f"xml error {e}")

    def test_one_entry_dict_is_ami_dictionary(self):
        """require the attribute to be present but does not check value"""
        setup_dict = AmiDictionaryTest().create_file_dict()
        one_dict = setup_dict[ONE_ENTRY_DICT]
        assert type(one_dict) is AmiDictionary, f"fila is not AmiDictionary {one_dict}"

    def test_dict1_has_version_attribute(self):
        """require the version attribute to be present but does not check value"""
        setup_dict = AmiDictionaryTest().create_file_dict()
        one_dict = setup_dict[DICTFILE1]
        amidict = AmiDictionary.create_from_xml_file(Path(one_dict))
        version = amidict.get_version()
        assert version == STARTING_VERSION

    def test_dict1_with_missing_version_attribute_is_not_valid(self):
        """
        require the version attribute to have starting value
        """
        setup_dict = AmiDictionaryTest().create_file_dict()
        amidict = AmiDictionary.create_from_xml_file(Path(setup_dict[DICTFILE1]))
        version = amidict.get_version()
        assert version == STARTING_VERSION

    def test_one_entry_dict_has_version_attribute(self):
        """require the attribiute to be present but does not check value"""
        setup_dict = AmiDictionaryTest().create_file_dict()
        one_dict = setup_dict[ONE_ENTRY_DICT]
        assert one_dict is not None
        version = one_dict.get_version()
        assert version == "1.2.3"

    def test_dictionary_has_version(self):
        """require the attribute to be present but does not check value"""
        setup_dict = AmiDictionaryTest().create_file_dict()
        one_dict = setup_dict[ONE_ENTRY_DICT]
        version = one_dict.get_version()
        assert version is not None, "missing version"

class DictionaryCreationTest(AmiAnyTest):
    """
    test methods which create dictionaries
    (from lists of strings, files, urls, sparql etc.

    """

    def test_read_wellformed_dictionary(self):
        """test can create from XML string
        includes well-formed and non-well-formed XML
        """
        dict_str = """
        <dictionary title='foo'>
        </dictionary>
        """
        ami_dict = AmiDictionary.create_dictionary_from_xml_string(dict_str)
        assert ami_dict is not None

        assert ami_dict.root.tag == "dictionary"

        dict_str = """
        <diktionary title='foo'>
        </dictionary>
        """
        try:
            ami_dict = AmiDictionary.create_dictionary_from_xml_string(dict_str)
        except XMLSyntaxError as e:
            logger.warning(f"xml error {e}")

    def test_dictionary_creation(self):
        """
        unit test
        """
        amidict = AmiDictionary.create_minimal_dictionary()
        assert amidict is not None
        assert amidict.get_version() == STARTING_VERSION

    def test_create_dictionary_from_list_of_strings_in_file(dictionary
#                                                                         , words_txt, entry_count
                                                                         ):
        """
        read file with lists of strings and create AmiDictionary
        """
        dictionary = "climate"
        words_txt = "ar5_wg3_food_security_words.txt"
        entry_count = 60
        dictionary_in_dir = Path(Resources.TEST_RESOURCES_DIR, "dictionary")
        out_dict_dir = Path(Resources.TEMP_DIR, "dictionary", "climate")
        words_file = Path(dictionary_in_dir, dictionary, words_txt)
        assert words_file.exists(), f"{words_file} should exist"
# create dictionary without save or lookup
        words_dictionary, _ = AmiDictionary.create_dictionary_from_wordfile(wordfile=words_file)
        assert words_dictionary is not None, (f"words dictionary from {words_file} must not be None")
        assert words_dictionary.get_entry_count() == entry_count, f"should have {entry_count} entries , found {words_dictionary.get_entry_count()}"


    def test_createdictionary_from_list_of_strings_in_file_and_lookup_wikidata(self):
# add title, description, and save
        out_dict_dir = Path(Resources.TEMP_DIR, "dictionary", "climate")
        words_file = Path(Resources.TEST_RESOURCES_DIR, "dictionary", "climate",
                          "ar5_wg3_food_security_words.txt")
        max_entries = 2 # to reduce time

        description = "food security terms in AR5_WG3_ch_5 selected by Anmol Negi"
        # add title, description, and save
        description = "food security terms in AR5_WG3_ch_5 selected by Anmol Negi and wikidata lookup"
        words_dictionary, outfile = AmiDictionary.create_dictionary_from_wordfile(
            wordfile=words_file, desc=description, title="food_security", outdir=out_dict_dir, wikidata=True,
            max_entries=max_entries, debug=True)
        assert words_dictionary is not None, f"words dictionary from {words_file} must not be None"
        assert words_dictionary.get_entry_count() == max_entries, \
            f"should have {max_entries} entries , found {words_dictionary.get_entry_count()}"
        assert outfile.exists(), f"should create {outfile}"

    def test_process_args_build_dictionary_add_wikidata_CMD_OK(self):
        """commandline
        reads list of words, creates a dictionary, looks up Wikidata
        Wikidata has <entry>s in in format:
    <entry name="UNFCCC" term="UNFCCC">
    <div qid="Q21707860">
        <a href="wikidataURL/Q21707860">Wikidata</a>
        <p role="title">Paris Agreement</p>
        <p role="description">international agreement from 12 December 2015 within the United Nations Framework Convention on Climate Change (UNFCCC) dealing with greenhouse gas emissions mitigation, adaptation and finance starting in the year 2020</p>
    </div>
    <div qid="Q208645">
        <a href="wikidataURL/Q208645">Wikidata</a>
        <p role="title">United Nations Framework Convention on Climate Change</p>
        <p role="description">international treaty</p>
    </div>
    ...

        Note that wikidata lookup does not necessarily put the best answers first
        """
        amilib = AmiLib()
        title = "small_2"
        title_path = Path(Resources.TEST_RESOURCES_DIR, "wordlists", f"{title}.txt")
        assert title_path.exists(), f"{title_path} should exist"
        title_txt = f"{title_path}"
        logger.debug(f"words {title_txt}")
        with open(title_txt, "r") as f:
            logger.debug(f"words ==> {f.readlines()}")
        title_dict = Path(Resources.TEMP_DIR, "words", f"{title}_dict.xml")
        FileLib.delete_file(title_dict)

        word_xml = f"{Path(Resources.TEMP_DIR, '{title}.xml')}"
        args = ["DICT",
                "--words", f"{title_path}",
                "--dict", f"{title_dict}",
                "--wikidata"
                ]
        logger.debug(f" s2 {args}")
        amilib.run_command(args)
        assert Path(title_dict).exists(), f"should write dictionary {title_dict}"
        print(f"wrote {title_dict}")

    # integrations
    def test_create_dictionary_from_list_of_string(self):
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        title = "foobar"
        directory = None
        amidict, _ = AmiDictionary.create_dictionary_from_words(terms, title)
        assert amidict is not None
        title = amidict.root.attrib[TITLE]
        assert title == "foobar"

    def test_create_dictionary_from_list_of_string_and_save(self):
        """
        Create dictionary from list of strings (no lookup)
        """
        terms = ["acetone", "benzene", "chloroform", "DMSO", "ethanol"]
        title = "solvents"
        temp_dir = Path(AmiAnyTest.TEMP_DIR, "dictionary")
        temp_dir.mkdir(exist_ok=True, parents=True)
        amidict, dictfile = AmiDictionary.create_dictionary_from_words(terms, title=title, outdir=temp_dir)
        outfile = Path(temp_dir, title + ".xml")
        assert Path(dictfile) == Path(temp_dir, title + ".xml") and os.path.exists(dictfile)
        assert amidict.get_entry_count() == 5

    def test_make_dict_from_file_CMD_OK(self):

        amilib = AmiLib()
        words_file = Path(TEST_RESOURCE_DIR, "wordlists", "climate_words.txt")
        dictfile = Path(Resources.TEMP_DIR, "words", "climate_words.xml")
        expected_count = 11
        FileLib.delete_file(dictfile)
        amilib.run_command(["DICT",
                            "--words", words_file,
                            "--dict", dictfile
                            ])
        assert Path(dictfile).exists()
        ami_dictionary = AmiDictionary.create_from_xml_file(dictfile)
        assert (c := ami_dictionary.get_entry_count()) == expected_count, \
            f"dictionary should contain {expected_count} found {c}"

    def test_make_dict_from_words_CMD_OK(self):
        amilib = AmiLib()
        words_file = Path(TEST_RESOURCE_DIR, "wordlists", "climate_words.txt")
        dictfile = Path(Resources.TEMP_DIR, "words", "climate_words.xml")
        expected_count = 1
        FileLib.delete_file(dictfile)
        amilib.run_command(["DICT",
                            "--words", "bread",
                            "--dict", dictfile
                            ])
        assert Path(dictfile).exists()
        ami_dictionary = AmiDictionary.create_from_xml_file(dictfile)
        assert (c := ami_dictionary.get_entry_count()) == expected_count, \
            f"dictionary should contain {expected_count} found {c}"

    def test_make_dict_from_words(self):
        """
        filas on "kangaroo"
        Reason unknown
        """
        amilib = AmiLib()
        dictfile = Path(Resources.TEMP_DIR, "words", "kangaroo.xml")
        expected_count = 1
        FileLib.delete_file(dictfile)
        amilib.run_command(["DICT",
                            "--words", "kangaroo",
                            "--dict", dictfile,
                            ])
        assert Path(dictfile).exists()
        ami_dictionary = AmiDictionary.create_from_xml_file(dictfile)
        assert (c := ami_dictionary.get_entry_count()) >= expected_count, \
            f"dictionary should contain {expected_count} found {c}"

    def test_create_dictionary_from_csv(self):
        """This does lookup, unfortunately Wikidata lookup changes as terms are added so this test is a mess"""
        csv_term_file = Path(Resources.TEST_IPCC_CHAP08_DICT, "urban_terms_1.csv")
        assert os.path.exists(str(csv_term_file)), f"csv_term_file should exist {csv_term_file}"
        keyword_dict = AmiDictionary.create_dictionary_from_csv(csv_term_file, col_name='keyword/phrase',
                                                                title="urban_terms_1")
        print(f"keyword_dict {keyword_dict}")
        entry = keyword_dict.get_first_ami_entry()
        assert entry.get_term() == "urbanization"
        assert entry.get_wikidata_id() == "Q161078" or entry.get_wikidata_id() == 'Q1508'
        assert len(keyword_dict.entries) == 1
        root_elem = keyword_dict.root
        assert root_elem.tag == 'dictionary'
        entry0 = root_elem.xpath("entry")[0]
        assert entry0.attrib["name"] == 'urbanization'
        # assert str(root) == \
        #        '<dictionary title="urban_terms_1"><entry name="urbanization" term="urbanization" wikidataID="Q161078" wikidataURL="https://www.wikidata.org/entity/Q161078" desc="longterm population movements (shift) from rural to urban areas;gradual increase in the proportion of people living in urban areas, and the ways in which each society adapts to the change;process by which towns and cities are formed and become larger"><wikidataHit type="wikidata_hits">Q5381005</wikidataHit><wikidataHit type="wikidata_hits">Q23580084</wikidataHit><wikidataHit type="wikidata_hits">Q2608153</wikidataHit><wikidataHit type="wikidata_hits">Q95443723</wikidataHit></entry></dictionary>'

    def test_validate_linked_dict(self):
        """
        tests a veriety of exploratory concepts such as inter- and intra-dictionary links
        DEVELOP 2022-12
        """
        dict_path = Path(Resources.TEST_IPCC_DICT_DIR, "raw_linked_dict.xml")
        assert dict_path.exists(), f"{dict_path} should exist"
        dictx = AmiDictionary.create_from_xml_file(dict_path)
        assert dictx is not None
        assert dictx.get_entry_count() == 17, f"dictionary has {dictx.get_entry_count()} entries"
        assert dictx.get_version() == '0.0.5', f"dictionary has version {dictx.get_version()}"
        terms = dictx.get_terms()
        assert len(terms) > 0, f"found {len(terms)}"
        assert terms == ['carbon budget',
                         'ffi',
                         'folu',
                         'fluorinated gas',
                         'carbon pricing',
                         'edgar',
                         'cait',
                         'ceds',
                         'crf',
                         'emissions factor',
                         'emission inventory',
                         'emission sectors',
                         'gwp100',
                         'slcf',
                         'cgtp',
                         'co2-equivalent emission',
                         'baseline scenario'], f'found {terms}'
    def test_create_dictionary_from_sparql(self):
        """
        read spqrql file and create dictionary
        then update from another sparql file
        """
        # PLANT = os.path.join(PHYSCHEM_RESOURCES, "plant")
        PLANT = os.path.join(Resources.TEST_RESOURCES_DIR, "plant")
        sparql_file = os.path.join(PLANT, "plant_part_sparql.xml")
        dictionary_file = os.path.join(PLANT, "eoplant_part.xml")
        assert Path(sparql_file).exists(), f"sparql_file should exist {sparql_file}"
        assert Path(dictionary_file).exists(), f"dictionary_file should exist {dictionary_file}"
        """
        <result>
            <binding name='item'>
                <uri>http://www.wikidata.org/entity/Q2923673</uri>
            </binding>
            <binding name='image'>
                <uri>http://commons.wikimedia.org/wiki/Special:FilePath/White%20Branches.jpg</uri>
            </binding>
        </result>
"""
        sparql_to_dictionary = {
            "id_name": "item",
            "sparql_name": "image",
            "dict_name": "image",
        }
        dictionary = AmiDictionary.create_from_xml_file(dictionary_file)
        wikidata_sparql = WikidataSparql(dictionary)
        wikidata_sparql.update_from_sparql(sparql_file, sparql_to_dictionary)
        outdir = Path(AmiAnyTest.TEMP_DIR, "sparql")
        outdir.mkdir(exist_ok=True)
        dictionary.write_to_dir(outdir)
        outpath = Path(outdir, "eoplant_part.xml")
        assert outpath.exists(), f"{outpath} should have been written"

    def test_create_dictionary_from_list_of_string_and_add_wikidata(self):
        terms = ["acetone",
                 "chloroform",
                 # "DMSO",
                 # "ethanol"
                 ]
        amidict, _ = AmiDictionary.create_dictionary_from_words(terms, title="solvents", wikidata=True)
        temp_dir = Path(AmiAnyTest.TEMP_DIR, "dictionary")
        dictfile = amidict.write_to_dir(temp_dir)

        with open(dictfile, "r") as f:
            dict_text = f.read()
        dict_text = re.sub("date=\"[^\"]*\"", "date=\"TODAY\"", dict_text)

        # note, the date is stripped as it changes with each run
        text1 = """<dictionary version="0.0.1" title="solvents" encoding="UTF-8">
      <metadata user="pm286" date="TODAY"/>
      <entry term="acetone" wikidataID="Q49546" description="chemical compound"/>
      <entry term="chloroform" wikidataID="Q172275" description="chemical compound"/>
    </dictionary>
    """
        # assert text1 == dict_text, f"{text1} != {dict_text}"
        # TODO remove user from metadata

    def test_create_dictionary_from_list_of_string(self):
        terms = ["foo", "bar", "plugh", "xyzzy", "baz"]
        title = "foobar"
        directory = None
        amidict, _ = AmiDictionary.create_dictionary_from_words(terms, title)
        assert amidict is not None
        title = amidict.root.attrib[TITLE]
        assert title == "foobar"

    def test_create_dictionary_from_list_of_string_and_save(self):
        """
        Create dictionary from list of strings (no lookup)
        """
        terms = ["acetone", "benzene", "chloroform", "DMSO", "ethanol"]
        title = "solvents"
        temp_dir = Path(AmiAnyTest.TEMP_DIR, "dictionary")
        temp_dir.mkdir(exist_ok=True, parents=True)
        amidict, dictfile = AmiDictionary.create_dictionary_from_words(terms, title=title, outdir=temp_dir)
        outfile = Path(temp_dir, title + ".xml")
        assert Path(dictfile) == Path(temp_dir, title + ".xml") and os.path.exists(dictfile)
        assert amidict.get_entry_count() == 5

    def test_create_dictionary_from_list_of_string_save_and_compare(self):
        terms = ["acetone", "benzene", "chloroform", "DMSO", "ethanol"]
        temp_dir = Path(AmiAnyTest.TEMP_DIR, "dictionary")
        amidict, dictfile = AmiDictionary.create_dictionary_from_words(terms, title="solvents", outdir=temp_dir)
        with open(dictfile, "r") as f:
            dict_text = f.read()
        dict_text = re.sub("date=\"[^\"]*\"", "date=\"TODAY\"", dict_text)
        assert len(dict_text) > 200, "lines of dict_text"
        assert type(dict_text) is str, f"{type(dict_text)}"
        # note, the date is nstripped as it changes with each run
        text1 = """<dictionary version="0.0.1" title="solvents" encoding="UTF-8">
      <metadata user="pm286" date="TODAY"/>
      <entry term="acetone"/>
      <entry term="benzene"/>
      <entry term="chloroform"/>
      <entry term="DMSO"/>
      <entry term="ethanol"/>
    </dictionary>
    """
        # assert text1 == dict_text, f"{text1} != {dict_text}"
        # TODO remove user from metadata

    def test_create_dictionary_from_url(self):
        """lookup entry in Github repository
        note: depends on INTERNET"""
        mentha_url = "https://raw.githubusercontent.com/petermr/pyami/main/py4ami/resources/amidicts/mentha_tps.xml"
        mentha_dict = AmiDictionary.create_dictionary_from_url(mentha_url)
        assert len(mentha_dict.get_lxml_entries()) == 1
        assert mentha_dict.get_first_ami_entry().get_term() == "1,8-cineole synthase"
        assert mentha_dict.get_version() == "0.0.3"

    def test_make_dict_from_words_and_add_wikimedia_image(self):
        """
        Reason unknown
        """
        amilib = AmiLib()
        dictfile = Path(Resources.TEMP_DIR, "words", "kangaroo.xml")
        expected_count = 1
        FileLib.delete_file(dictfile)
        amilib.run_command(["DICT",
                            "--words", "kangaroo",
                            "--dict", dictfile,
                            ])
        assert Path(dictfile).exists()
        ami_dictionary = AmiDictionary.create_from_xml_file(dictfile)
        assert (c := ami_dictionary.get_entry_count()) >= expected_count, \
            f"dictionary should contain {expected_count} found {c}"

    def test_make_dict_from_wordlist_add_figures_LONG(self):
        stems = [
            # "breward",
            # "carbon_cycle_noabb",
            # "chap2",
            # "chap5",
            # "chap6_7_8",
            # "food_ecosystem",
            # "human_influence",
            # "inews15",
            "mini3",
            # "non_wikipedia",
            # "poverty",
            # "small_2",
            # "water_cyclone"
            ""

            ]

        for stem in stems:
            logger.info(f"creating figure entries for {stem}")
            self.create_dict_for_word_file(stem, wiktionary=True)

    def create_dict_for_word_file(
            self, stem, wiktionary=True, input_file=None, output_dict=None):
        """
        Reads a wordlist file and creates output dictionary with figures
        :param stem: stem of files, also used as title
        :param input_file: if None, defaults to TEST_RESOURCES_DIR, "wordlists", f"{stem}.txt"
        :param output_dict: if None, defaults to TEMP_DIR, "words", "html", f"{stem}.html"

        """
        if stem is None or stem.strip() == "":
            logger.error("no title given for dictionary")
            return None
        if input_file is None:
            input_file = Path(Resources.TEST_RESOURCES_DIR, "wordlists", f"{stem}.txt")
        if output_dict is None:
            output_dict = Path(Resources.TEMP_DIR, "words", "html", f"{stem}.html")

        args = [
            "DICT",
            "--words", input_file,
            "--dict", output_dict,
            "--figures", "wikipedia",
            "--title", stem,
        ]
        if wiktionary:
            args.append("--wiktionary")

        AmiLib().run_command(args)
        assert Path(output_dict).exists()

    def test_make_dict_with_figures(self):
        stem = "with_figures1"
        output_dict = Path(Resources.TEMP_DIR, "words", "html", f"{stem}.html")

        words = [
            "albedo",
            "ablation",
        ]
        args = [
            "DICT",
            "--words", words,
            "--dict", output_dict,
            "--title", stem,
            "--figures", "wikipedia",
        ]
        AmiLib().run_command(args)
        assert Path(output_dict).exists()

    def test_disambiguation(self):
        stem = ("disambiguation")
        output_dict = Path(Resources.TEMP_DIR, "words", "html", f"{stem}.html")

        args = [
            "DICT",
            "--words",
            "stubble",
            "anthropogenic",
            "--dict", output_dict,
            # "--figures",
        ]
        AmiLib().run_command(args)
        assert Path(output_dict).exists()

    def test_create_dict_with_wikimedia_descriptions(self):
        """
        from commandline add descriptions to ditionary entries from wikimedia sources
        """
        stem = "parijat"
        output_dict = Path(Resources.TEMP_DIR, "words", "html", f"{stem}.xml")

        args = [
            "DICT",
            "--words",
            stem,
            "--description",
            "wikipedia",
            "wikidata",
            "wiktionary",
            "--dict", output_dict,
        ]
        AmiLib().run_command(args)
        assert Path(output_dict).exists()

class AmiIndexTest(AmiAnyTest):
    """
    tests to develop a book-like index of words
    """

    def test_index_pages_by_words_LONG(self):
        """
        read a list of pages, and the words in them and
        create a simple index pointing to occurrence of words in pages
        """
        # TODO split into smaller methods
# import urllib.request  # the lib that handles the url stuff

        # Counter (words, count) created for words in Climate Academy
        counter_url = "https://raw.githubusercontent.com/semanticClimate/internship_sC/refs/heads/shabnam/Wordlist_management/wordlistCa.txt"
        # repository for 357 text pages from Climate
        pages_url = "https://raw.githubusercontent.com/semanticClimate/internship_sC/refs/heads/MEBIN/Climate_Academy/Individual_Pages"
        counter = AmiIndexMisc.read_counter(counter_url)
        if counter is None:
            logger.error("No counter read")
            return
        print(f"counter {len(counter)}")
        max_page = 10 # to limit time
        # max_page = 357
        # max_page = 5
        # start = 200
        start = 1
        min_count = 1
        stopwords = {"The", "Academy", "is"} # don't understand just these

        google_response = requests.get("https://google.com")
        print(f"google {google_response}")
        files_response = requests.get(pages_url + "/")
        print(f"files {pages_url} {files_response}")
        html = AmiIndexMisc.create_html_index_of_pages(
            counter, max_page, min_count, pages_url, start, stopwords)

        index_file = Path(Resources.TEMP_DIR, "html", "climate_book", "index.html")
        HtmlLib.write_html_file(html, index_file, debug=True)

    def test_download_github_dir0(self):
        """
        Doesnt yet work
        """
        import os
        import requests

        # Set the GitHub repository info
        owner = "petermr"
        repo = "amilib"
        directory = "directory"  # The directory you want to list the files from

        # GitHub API URL for repository contents
        url = f"https://api.github.com/repos/{owner}/{repo}/contents/{directory}"
        url = f"https://api.github.com/repos/{owner}/{repo}/tree/pmr_aug/test/chapter11"

        # If you are using authentication (optional)
        headers = {
            "Authorization": "Bearer YOUR_GITHUB_TOKEN"
        }
        headers = {}

        # Function to download a file
        def download_file(file_url, file_name):
            response = requests.get(file_url)
            response.raise_for_status()  # Check for request errors

            with open(file_name, 'wb') as file:
                file.write(response.content)
            print(f"Downloaded {file_name}")

        # Fetch the directory contents
        response = requests.get(url, headers=headers)
        if response.status_code == 404:
            logger.error(f"404 cannot find {url}")
            return
        response.raise_for_status()

        # Iterate over the files in the directory
        files = response.json()

        for file in files:
            if file['type'] == 'file':  # Only files (not subdirectories)
                file_url = file['download_url']
                file_name = file['name']
                download_file(file_url, file_name)

    # def test_github_repos(self):
    #     pass
        # # Authentication is defined via github.Auth
        # from github import Auth
        #
        # # using an access token
        # auth = Auth.Token("access_token")
        #
        # # First create a Github instance:
        #
        # # Public Web Github
        # g = Github(auth=auth)
        #
        # # Github Enterprise with custom hostname
        # g = Github(base_url="https://{hostname}/api/v3", auth=auth)
        #
        # # Then play with your Github objects:
        # for repo in g.get_user().get_repos():
        #     print(repo.name)
        #
        # # To close connections after use
        # g.close()

class IPCCDictTest(AmiAnyTest):
    def test_ipcc_dictionaries_from_URL(self):
        """
        read a number of IPCC dictionaries from URL and test their validity.
        Maybe later move code to dictionary project
        """
        WG3_ROOT = "https://raw.githubusercontent.com/petermr/semanticClimate/main/ipcc/ar6/wg3"
        dictionary_names = [
            "Chapter02/dict/ip_3_2_emissions_abb.xml", "Chapter07/dict/ip_3_7_agri_abb.xml",
            "wg2_03/dict/ip_3_3_longmitig_abb.xml", "Chapter08/dict/ip_3_8_urban_abb.xml",
            "Chapter05/dict/ip_3_5_socmitig_abb.xml", "Chapter17/dict/ip_3_17_sustdev_abb.xml",
            "Chapter06/dict/ip_3_6_energy_abb.xml"
        ]

        for dictionary_name in dictionary_names:
            url = WG3_ROOT + "/" + dictionary_name
            print(f" url {url}")
            try:
                dictionary = AmiDictionary.create_dictionary_from_url(url)
            except Exception as e:
                print(f"cannot download dictionary {url} beacuse {e}")

# ======================= helpers =====================
    def teardown(self):
        dict1_root = None

def main(argv=None):
    print(f"running PDFArgs main")
    pdf_args = AmiDictArgs()
    try:
        pdf_args.parse_and_process()
    except Exception as e:
        print(traceback.format_exc())
        print(f"***Cannot run pyami***; see output for errors: {e}")

if __name__ == "__main__":
    main()
else:
    pass

def main():
    pass

class AmiDictionaryJSONTest(AmiAnyTest):
    """Tests for JSON serialization/deserialization of AmiDictionary and AmiEntry"""
    
    def setUp(self):
        """Set up test fixtures"""
        self.test_dict = AmiDictionary.create_minimal_dictionary()
        self.test_dict.set_title("Test Dictionary")
        self.test_dict.set_version("1.0.0")
        
        # Add test entries
        entry1_element = self.test_dict.create_and_add_entry_with_term("climate change")
        entry1 = AmiEntry.create_from_element(entry1_element)
        entry1.set_description("Long-term alteration of temperature patterns")
        entry1.set_wikidata_id("Q125928")
        
        entry2_element = self.test_dict.create_and_add_entry_with_term("global warming")
        entry2 = AmiEntry.create_from_element(entry2_element)
        entry2.set_description("Rise in global average temperature")
        entry2.set_wikidata_id("Q7942")
    
    def test_ami_entry_to_json(self):
        """Test converting AmiEntry to JSON"""
        entry = self.test_dict.get_ami_entry("climate change")
        json_data = entry.to_json()
        
        # Check required fields
        self.assertIn("term", json_data, "JSON should contain 'term' field")
        self.assertEqual(json_data["term"], "climate change", "Term should match expected value")
        
        # Check optional fields
        self.assertIn("description", json_data, "JSON should contain 'description' field")
        self.assertEqual(json_data["description"], "Long-term alteration of temperature patterns", "Description should match expected value")
        self.assertIn("wikidata_id", json_data, "JSON should contain 'wikidata_id' field")
        self.assertEqual(json_data["wikidata_id"], "Q125928", "Wikidata ID should match expected value")
        
        # Check structure
        self.assertIsInstance(json_data, dict, "JSON data should be a dictionary")
        self.assertIn("id", json_data, "JSON should contain 'id' field")
    
    def test_ami_entry_from_json(self):
        """Test creating AmiEntry from JSON"""
        json_data = {
            "term": "test term",
            "description": "test description",
            "wikidata_id": "Q12345",
            "name": "test name"
        }
        
        entry = AmiEntry()
        entry.from_json(json_data)
        
        self.assertEqual(entry.get_term(), "test term", "Term should be set correctly from JSON")
        self.assertEqual(entry.get_description(), "test description", "Description should be set correctly from JSON")
        self.assertEqual(entry.get_wikidata_id(), "Q12345", "Wikidata ID should be set correctly from JSON")
        self.assertEqual(entry.get_name(), "test name", "Name should be set correctly from JSON")
    
    def test_ami_dictionary_to_json(self):
        """Test converting AmiDictionary to JSON"""
        json_data = self.test_dict.to_json()
        
        # Check metadata
        self.assertIn("metadata", json_data, "JSON should contain 'metadata' section")
        metadata = json_data["metadata"]
        self.assertEqual(metadata["title"], "Test Dictionary", "Title should match expected value")
        self.assertEqual(metadata["version"], "1.0.0", "Version should match expected value")
        self.assertIn("entry_count", metadata, "Metadata should contain 'entry_count' field")
        self.assertEqual(metadata["entry_count"], 2, "Entry count should match expected value")
        
        # Check entries
        self.assertIn("entries", json_data, "JSON should contain 'entries' section")
        entries = json_data["entries"]
        self.assertEqual(len(entries), 2, "Should have exactly 2 entries")
        
        # Check first entry
        entry1 = entries[0]
        self.assertEqual(entry1["term"], "climate change", "First entry term should match expected value")
        self.assertEqual(entry1["description"], "Long-term alteration of temperature patterns", "First entry description should match expected value")
        self.assertEqual(entry1["wikidata_id"], "Q125928", "First entry Wikidata ID should match expected value")
    
    def test_ami_dictionary_from_json(self):
        """Test creating AmiDictionary from JSON"""
        json_data = {
            "metadata": {
                "title": "Test Dictionary",
                "version": "1.0.0",
                "language": "en",
                "description": "Test dictionary for JSON functionality"
            },
            "entries": [
                {
                    "term": "test term 1",
                    "description": "test description 1",
                    "wikidata_id": "Q12345"
                },
                {
                    "term": "test term 2",
                    "description": "test description 2",
                    "wikidata_id": "Q67890"
                }
            ]
        }
        
        dictionary = AmiDictionary()
        dictionary.from_json(json_data)
        
        # Check metadata
        self.assertEqual(dictionary.get_title(), "Test Dictionary", "Title should be set correctly from JSON")
        self.assertEqual(dictionary.get_version(), "1.0.0", "Version should be set correctly from JSON")
        
        # Check entries
        self.assertEqual(dictionary.get_entry_count(), 2, "Entry count should be set correctly from JSON")
        
        entry1 = dictionary.get_ami_entry("test term 1")
        self.assertIsNotNone(entry1, "Entry should be found by term")
        self.assertEqual(entry1.get_description(), "test description 1", "Entry description should be set correctly from JSON")
        self.assertEqual(entry1.get_wikidata_id(), "Q12345", "Entry Wikidata ID should be set correctly from JSON")
    
    def test_ami_dictionary_save_and_load_json(self):
        """Test saving and loading AmiDictionary as JSON file"""
        import tempfile
        import os
        
        with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
            temp_file = f.name
        
        try:
            # Save dictionary to JSON
            self.test_dict.save_json(temp_file)
            
            # Verify file exists and has content
            self.assertTrue(os.path.exists(temp_file), "JSON file should be created")
            with open(temp_file, 'r') as f:
                content = f.read()
                self.assertGreater(len(content), 0, "JSON file should not be empty")
            
            # Load dictionary from JSON
            loaded_dict = AmiDictionary.load_json(temp_file)
            
            # Verify loaded dictionary matches original
            self.assertEqual(loaded_dict.get_title(), self.test_dict.get_title(), "Loaded title should match original")
            self.assertEqual(loaded_dict.get_version(), self.test_dict.get_version(), "Loaded version should match original")
            self.assertEqual(loaded_dict.get_entry_count(), self.test_dict.get_entry_count(), "Loaded entry count should match original")
            
            # Verify entries match
            original_entry = self.test_dict.get_ami_entry("climate change")
            loaded_entry = loaded_dict.get_ami_entry("climate change")
            self.assertEqual(original_entry.get_description(), loaded_entry.get_description(), "Loaded entry description should match original")
            self.assertEqual(original_entry.get_wikidata_id(), loaded_entry.get_wikidata_id(), "Loaded entry Wikidata ID should match original")
            
        finally:
            # Clean up
            if os.path.exists(temp_file):
                os.unlink(temp_file)
    
    def test_json_roundtrip(self):
        """Test JSON serialization and deserialization roundtrip"""
        # Convert to JSON
        json_data = self.test_dict.to_json()
        
        # Convert back to dictionary
        new_dict = AmiDictionary()
        new_dict.from_json(json_data)
        
        # Verify they match
        self.assertEqual(self.test_dict.get_title(), new_dict.get_title(), "Title should be preserved in roundtrip")
        self.assertEqual(self.test_dict.get_version(), new_dict.get_version(), "Version should be preserved in roundtrip")
        self.assertEqual(self.test_dict.get_entry_count(), new_dict.get_entry_count(), "Entry count should be preserved in roundtrip")
        
        # Verify entries match
        original_terms = set(self.test_dict.get_terms())
        new_terms = set(new_dict.get_terms())
        self.assertEqual(original_terms, new_terms, "Terms should be preserved in roundtrip")
    
    def test_empty_dictionary_json(self):
        """Test JSON handling of empty dictionary"""
        empty_dict = AmiDictionary.create_minimal_dictionary()
        empty_dict.set_title("Empty Dictionary")
        empty_dict.set_version("1.0.0")
        
        json_data = empty_dict.to_json()
        
        self.assertIn("metadata", json_data, "Empty dictionary JSON should contain metadata section")
        self.assertEqual(json_data["metadata"]["entry_count"], 0, "Empty dictionary should have zero entry count")
        self.assertIn("entries", json_data, "Empty dictionary JSON should contain entries section")
        self.assertEqual(len(json_data["entries"]), 0, "Empty dictionary should have empty entries list")
    
    def test_json_with_missing_optional_fields(self):
        """Test JSON handling of entries with missing optional fields"""
        minimal_entry = AmiEntry()
        minimal_entry.add_term("minimal term")
        
        json_data = minimal_entry.to_json()
        
        # Required field should be present
        self.assertIn("term", json_data, "Minimal entry JSON should contain term field")
        self.assertEqual(json_data["term"], "minimal term", "Term should match expected value")
        
        # Optional fields should be None or missing
        self.assertNotIn("description", json_data, "Minimal entry should not contain description field")
        self.assertNotIn("wikidata_id", json_data, "Minimal entry should not contain wikidata_id field")
        self.assertNotIn("name", json_data, "Minimal entry should not contain name field")
