"""
Adding and removing lexicons to/from the database.
"""

from typing import (
    Union,
    Optional,
    TypeVar,
    Type,
    Tuple,
    List,
    Dict,
    Set,
    Iterator,
    Iterable,
    Sequence,
    cast,
)
from pathlib import Path
from itertools import islice
import sqlite3
import logging

import wn
from wn._types import AnyPath
from wn.constants import _WORDNET, _ILI
from wn._db import connect
from wn._queries import find_lexicons, get_lexicon_extensions, get_lexicon
from wn._util import normalize_form
from wn.util import ProgressHandler, ProgressBar
from wn.project import iterpackages
from wn import lmf
from wn import _ili


logger = logging.getLogger("wn")


BATCH_SIZE = 1000
DEFAULT_MEMBER_RANK = 127  # synset member rank when not specified by 'members'

ENTRY_QUERY = """
    SELECT e.rowid
      FROM entries AS e
     WHERE e.id = ?
       AND e.lexicon_rowid = ?
"""
# forms don't have reliable ids, so also consider rank; this depends
# on each form having a unique rank, and this doesn't work for lexicon
# extensions
FORM_QUERY = """
    SELECT f.rowid
      FROM forms AS f
      JOIN entries AS e ON f.entry_rowid = e.rowid
     WHERE e.id = ?
       AND e.lexicon_rowid = ?
       AND (f.id = ? OR f.rank = ?)
"""
SENSE_QUERY = """
    SELECT s.rowid
      FROM senses AS s
     WHERE s.id = ?
       AND s.lexicon_rowid = ?
"""
SYNSET_QUERY = """
    SELECT ss.rowid
      FROM synsets AS ss
     WHERE ss.id = ?
       AND ss.lexicon_rowid = ?
"""
RELTYPE_QUERY = """
    SELECT rt.rowid
      FROM relation_types AS rt
     WHERE rt.type = ?
"""
ILISTAT_QUERY = """
    SELECT ist.rowid
      FROM ili_statuses AS ist
     WHERE ist.status = ?
"""
LEXFILE_QUERY = """
    SELECT lf.rowid
      FROM lexfiles AS lf
     WHERE lf.name = ?
"""

_AnyLexicon = Union[lmf.Lexicon, lmf.LexiconExtension]
_AnyEntry = Union[lmf.LexicalEntry, lmf.ExternalLexicalEntry]
_AnyLemma = Union[lmf.Lemma, lmf.ExternalLemma]
_AnyForm = Union[lmf.Form, lmf.ExternalForm]
_AnySense = Union[lmf.Sense, lmf.ExternalSense]
_AnySynset = Union[lmf.Synset, lmf.ExternalSynset]


def add(
    source: AnyPath,
    progress_handler: Optional[Type[ProgressHandler]] = ProgressBar,
) -> None:
    """Add the LMF file at *source* to the database.

    The file at *source* may be gzip-compressed or plain text XML.

    >>> wn.add('english-wordnet-2020.xml')
    Added ewn:2020 (English WordNet)

    The *progress_handler* parameter takes a subclass of
    :class:`wn.util.ProgressHandler`. An instance of the class will be
    created, used, and closed by this function.

    """
    if progress_handler is None:
        progress_handler = ProgressHandler
    progress = progress_handler(message="Database")

    logger.info("adding project to database")
    logger.info("  database: %s", wn.config.database_path)
    logger.info("  project file: %s", source)

    try:
        for package in iterpackages(source):
            if package.type == _WORDNET:
                _add_lmf(package.resource_file(), progress, progress_handler)
            elif package.type == _ILI:
                _add_ili(package.resource_file(), progress)
            else:
                raise wn.Error(f"unknown package type: {package.type}")
    finally:
        progress.close()


def _add_lmf(
    source: Path,
    progress: ProgressHandler,
    progress_handler: Type[ProgressHandler],
) -> None:
    with connect() as conn:
        cur = conn.cursor()
        # these two settings increase the risk of database corruption
        # if the system crashes during a write, but they should also
        # make inserts much faster
        cur.execute("PRAGMA synchronous = OFF")
        cur.execute("PRAGMA journal_mode = MEMORY")

        # abort if any lexicon in *source* is already added
        progress.flash(f"Checking {source!s}")
        all_infos = _precheck(source, cur)

        if not all_infos:
            progress.flash(f"{source}: No lexicons found")
            return
        for info in all_infos:
            skip = info.get("skip", "")
            if skip:
                id, ver, lbl = info["id"], info["version"], info["label"]
                progress.flash(f"Skipping {id}:{ver} ({lbl}); {skip}\n")
        if all("skip" in info for info in all_infos):
            return

        # all clear, try to add them
        progress.flash(f"Reading {source!s}")
        resource = lmf.load(source, progress_handler)

        for lexicon, info in zip(resource["lexicons"], all_infos):
            if "skip" in info:
                continue

            progress.flash("Updating lookup tables")
            _update_lookup_tables(lexicon, cur)

            progress.set(count=0, total=_sum_counts(lexicon))
            synsets: Sequence[_AnySynset] = _synsets(lexicon)
            entries: Sequence[_AnyEntry] = _entries(lexicon)
            synbhrs: Sequence[lmf.SyntacticBehaviour] = _collect_frames(lexicon)

            lexid, extid = _insert_lexicon(lexicon, info, cur, progress)

            lexidmap = _build_lexid_map(lexicon, lexid, extid)

            _insert_synsets(synsets, lexid, cur, progress)
            _insert_entries(entries, lexid, cur, progress)
            _insert_forms(entries, lexid, lexidmap, cur, progress)
            _insert_pronunciations(entries, lexid, lexidmap, cur, progress)
            _insert_tags(entries, lexid, lexidmap, cur, progress)
            _insert_senses(entries, synsets, lexid, lexidmap, cur, progress)
            _insert_adjpositions(entries, lexid, lexidmap, cur, progress)
            _insert_counts(entries, lexid, lexidmap, cur, progress)
            _insert_syntactic_behaviours(synbhrs, lexid, lexidmap, cur, progress)

            _insert_synset_relations(synsets, lexid, lexidmap, cur, progress)
            _insert_sense_relations(lexicon, lexid, lexidmap, cur, progress)

            _insert_synset_definitions(synsets, lexid, lexidmap, cur, progress)
            _insert_examples(
                [sense for e in entries for sense in _senses(e)],
                lexid,
                lexidmap,
                "sense_examples",
                cur,
                progress,
            )
            _insert_examples(synsets, lexid, lexidmap, "synset_examples", cur, progress)

            progress.set(status="")  # clear type string
            progress.flash(
                f"Added {lexicon['id']}:{lexicon['version']} ({lexicon['label']})\n"
            )


def _precheck(source: Path, cur: sqlite3.Cursor) -> List[Dict]:
    lexqry = "SELECT * FROM lexicons WHERE id = :id AND version = :version"
    infos = lmf.scan_lexicons(source)
    for info in infos:
        base = info.get("extends")
        if cur.execute(lexqry, info).fetchone():
            info["skip"] = "already added"
        if base and cur.execute(lexqry, base).fetchone() is None:
            id_, ver = base["id"], base["version"]
            info["skip"] = f"base lexicon ({id_}:{ver}) not available"
    return infos


def _sum_counts(lex: _AnyLexicon) -> int:
    ents = _entries(lex)
    locs = _local_entries(ents)
    lems = [e["lemma"] for e in locs if e.get("lemma")]
    frms = [f for e in ents for f in _forms(e)]
    sens = [s for e in ents for s in _senses(e)]
    syns = _synsets(lex)
    return sum(
        [
            # lexical entries
            len(ents),
            len(lems),
            sum(len(lem.get("pronunciations", [])) for lem in lems),
            sum(len(lem.get("tags", [])) for lem in lems),
            len(frms),
            sum(len(frm.get("pronunciations", [])) for frm in frms),
            sum(len(frm.get("tags", [])) for frm in frms),
            # senses
            len(sens),
            sum(len(sen.get("relations", [])) for sen in sens),
            sum(len(sen.get("examples", [])) for sen in sens),
            sum(len(sen.get("counts", [])) for sen in sens),
            # synsets
            len(syns),
            sum(len(syn.get("definitions", [])) for syn in syns),
            sum(len(syn.get("relations", [])) for syn in syns),
            sum(len(syn.get("examples", [])) for syn in syns),
            # syntactic behaviours
            sum(len(ent.get("frames", [])) for ent in locs),
            len(lex.get("frames", [])),
        ]
    )


def _update_lookup_tables(lexicon: _AnyLexicon, cur: sqlite3.Cursor) -> None:
    reltypes = set(
        rel["relType"] for ss in _synsets(lexicon) for rel in ss.get("relations", [])
    )
    reltypes.update(
        rel["relType"]
        for e in _entries(lexicon)
        for s in _senses(e)
        for rel in s.get("relations", [])
    )
    cur.executemany(
        "INSERT OR IGNORE INTO relation_types VALUES (null,?)",
        [(rt,) for rt in sorted(reltypes)],
    )
    lexfiles: Set[str] = {
        ss.get("lexfile", "")
        for ss in _local_synsets(_synsets(lexicon))
        if ss.get("lexfile")
    }
    cur.executemany(
        "INSERT OR IGNORE INTO lexfiles VALUES (null,?)",
        [(lf,) for lf in sorted(lexfiles)],
    )


def _insert_lexicon(
    lexicon: _AnyLexicon, info: dict, cur: sqlite3.Cursor, progress: ProgressHandler
) -> Tuple[int, int]:
    progress.set(status="Lexicon Info")
    cur.execute(
        "INSERT INTO lexicons VALUES (null,?,?,?,?,?,?,?,?,?,?,?)",
        (
            lexicon["id"],
            lexicon["label"],
            lexicon["language"],
            lexicon["email"],
            lexicon["license"],
            lexicon["version"],
            lexicon.get("url"),
            lexicon.get("citation"),
            lexicon.get("logo"),
            lexicon.get("meta"),
            False,
        ),
    )
    lexid = cur.lastrowid

    if not isinstance(lexid, int):
        raise wn.Error("failed to insert lexicon")

    query = """
        UPDATE lexicon_dependencies
           SET provider_rowid = ?
         WHERE provider_id = ? AND provider_version = ?
    """
    cur.execute(query, (lexid, lexicon["id"], lexicon["version"]))

    query = """
        INSERT INTO {table}
        VALUES (:lid,
                :id,
                :version,
                :url,
                (SELECT rowid FROM lexicons WHERE id=:id AND version=:version))
    """
    params = []
    for dep in lexicon.get("requires", []):
        param_dict = dict(dep)
        param_dict.setdefault("url", None)
        param_dict["lid"] = lexid
        params.append(param_dict)
    if params:
        cur.executemany(query.format(table="lexicon_dependencies"), params)

    if lexicon.get("extends"):
        lexicon = cast(lmf.LexiconExtension, lexicon)
        param_dict = dict(lexicon["extends"])
        param_dict.setdefault("url", None)
        param_dict["lid"] = lexid
        cur.execute(query.format(table="lexicon_extensions"), param_dict)
        extid = cur.execute(
            "SELECT rowid FROM lexicons WHERE id=? AND version=?",
            (param_dict["id"], param_dict["version"]),
        ).fetchone()[0]
    else:
        extid = lexid

    return lexid, extid


_LexIdMap = Dict[str, int]


def _build_lexid_map(lexicon: _AnyLexicon, lexid: int, extid: int) -> _LexIdMap:
    """Build a mapping of entity IDs to extended lexicon rowid."""
    lexidmap: _LexIdMap = {}
    if lexid != extid:
        lexidmap.update((e["id"], extid) for e in _entries(lexicon) if _is_external(e))
        lexidmap.update(
            (s["id"], extid)
            for e in _entries(lexicon)
            for s in _senses(e)
            if _is_external(s)
        )
        lexidmap.update(
            (ss["id"], extid) for ss in _synsets(lexicon) if _is_external(ss)
        )
    return lexidmap


T = TypeVar("T")


def _batch(sequence: Iterable[T]) -> Iterator[List[T]]:
    it = iter(sequence)
    batch = list(islice(it, 0, BATCH_SIZE))
    while len(batch):
        yield batch
        batch = list(islice(it, 0, BATCH_SIZE))


def _insert_synsets(
    synsets: Sequence[_AnySynset],
    lexid: int,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Synsets")
    # synsets
    ss_query = f"""
        INSERT INTO synsets
        VALUES (null,?,?,(SELECT rowid FROM ilis WHERE id=?),?,?,({LEXFILE_QUERY}),?)
    """
    # presupposed ILIs
    pre_ili_query = f"""
        INSERT OR IGNORE INTO ilis
        VALUES (null,?,({ILISTAT_QUERY}),?,?)
    """
    # proposed ILIs
    pro_ili_query = """
        INSERT INTO proposed_ilis
        VALUES (null,
               (SELECT ss.rowid
                  FROM synsets AS ss
                 WHERE ss.id=? AND lexicon_rowid=?),
               ?,
               ?)
    """

    for batch in _batch(_local_synsets(synsets)):

        # first add presupposed ILIs
        pre_ili_data = []
        for ss in batch:
            ili = ss["ili"]
            if ili and ili != "in":
                defn = ss.get("ili_definition")  # normally null
                text = defn["text"] if defn else None
                meta = defn["meta"] if defn else None
                pre_ili_data.append((ili, "presupposed", text, meta))
        cur.executemany(pre_ili_query, pre_ili_data)

        # then add synsets
        ss_data = (
            (
                ss["id"],
                lexid,
                ss["ili"] if ss["ili"] and ss["ili"] != "in" else None,
                ss["partOfSpeech"],
                ss.get("lexicalized", True),
                ss.get("lexfile"),
                ss["meta"],
            )
            for ss in batch
        )
        cur.executemany(ss_query, ss_data)

        # finally add proposed ILIs
        pro_ili_data = []
        for ss in batch:
            ili = ss["ili"]
            if ili == "in":
                defn = ss.get("ili_definition")
                text = defn["text"] if defn else None
                meta = defn["meta"] if defn else None
                pro_ili_data.append((ss["id"], lexid, text, meta))
        cur.executemany(pro_ili_query, pro_ili_data)

        progress.update(len(batch))


def _insert_synset_definitions(
    synsets: Sequence[_AnySynset],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Definitions")
    query = f"""
        INSERT INTO definitions
        VALUES (null,?,({SYNSET_QUERY}),?,?,({SENSE_QUERY}),?)
    """
    for batch in _batch(synsets):
        data = [
            (
                lexid,
                synset["id"],
                lexidmap.get(synset["id"], lexid),
                definition["text"],
                definition.get("language"),
                definition.get("sourceSense"),
                lexidmap.get(definition.get("sourceSense", ""), lexid),
                definition["meta"],
            )
            for synset in batch
            for definition in synset.get("definitions", [])
        ]
        cur.executemany(query, data)
        progress.update(len(data))


def _insert_synset_relations(
    synsets: Sequence[_AnySynset],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Synset Relations")
    query = f"""
        INSERT INTO synset_relations
        VALUES (null,?,({SYNSET_QUERY}),({SYNSET_QUERY}),({RELTYPE_QUERY}),?)
    """
    for batch in _batch(synsets):
        data = [
            (
                lexid,
                synset["id"],
                lexidmap.get(synset["id"], lexid),
                relation["target"],
                lexidmap.get(relation["target"], lexid),
                relation["relType"],
                relation["meta"],
            )
            for synset in batch
            for relation in synset.get("relations", [])
        ]
        cur.executemany(query, data)
        progress.update(len(data))


def _insert_entries(
    entries: Sequence[_AnyEntry],
    lexid: int,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Words")
    query = "INSERT INTO entries VALUES (null,?,?,?,?)"
    for batch in _batch(_local_entries(entries)):
        data = (
            (entry["id"], lexid, entry["lemma"]["partOfSpeech"], entry["meta"])
            for entry in batch
        )
        cur.executemany(query, data)
        progress.update(len(batch))


def _insert_forms(
    entries: Sequence[_AnyEntry],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Word Forms")
    query = f"INSERT INTO forms VALUES (null,?,?,({ENTRY_QUERY}),?,?,?,?)"
    for batch in _batch(entries):
        forms: List[
            Tuple[Optional[str], int, str, int, str, Optional[str], Optional[str], int]
        ] = []
        for entry in batch:
            eid = entry["id"]
            lid = lexidmap.get(eid, lexid)
            if not _is_external(entry):
                entry = cast(lmf.LexicalEntry, entry)
                written_form = entry["lemma"]["writtenForm"]
                norm = normalize_form(written_form)
                forms.append(
                    (
                        None,
                        lexid,
                        eid,
                        lid,
                        written_form,
                        norm if norm != written_form else None,
                        entry["lemma"].get("script"),
                        0,
                    )
                )
            for i, form in enumerate(_forms(entry), 1):
                if _is_external(form):
                    continue
                form = cast(lmf.Form, form)
                written_form = form["writtenForm"]
                norm = normalize_form(written_form)
                forms.append(
                    (
                        form.get("id"),
                        lexid,
                        eid,
                        lid,
                        written_form,
                        norm if norm != written_form else None,
                        form.get("script"),
                        i,
                    )
                )
        cur.executemany(query, forms)
        progress.update(len(forms))


def _insert_pronunciations(
    entries: Sequence[_AnyEntry],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Pronunciations")
    query = f"INSERT INTO pronunciations VALUES (({FORM_QUERY}),?,?,?,?,?)"
    for batch in _batch(entries):
        prons: List[
            Tuple[
                str,
                int,
                Optional[str],
                int,
                str,
                Optional[str],
                Optional[str],
                bool,
                Optional[str],
            ]
        ] = []
        for entry in batch:
            eid = entry["id"]
            lid = lexidmap.get(eid, lexid)
            if entry.get("lemma"):
                for p in entry["lemma"].get("pronunciations", []):
                    prons.append(
                        (
                            eid,
                            lid,
                            None,
                            0,
                            p["text"],
                            p.get("variety"),
                            p.get("notation"),
                            p.get("phonemic", True),
                            p.get("audio"),
                        )
                    )
            for i, form in enumerate(_forms(entry), 1):
                # rank is not valid in FORM_QUERY for external forms
                rank = -1 if _is_external(form) else i
                for p in form.get("pronunciations", []):
                    prons.append(
                        (
                            eid,
                            lid,
                            form.get("id"),
                            rank,
                            p["text"],
                            p.get("variety"),
                            p.get("notation"),
                            p.get("phonemic", True),
                            p.get("audio"),
                        )
                    )
        cur.executemany(query, prons)
        progress.update(len(prons))


def _insert_tags(
    entries: Sequence[_AnyEntry],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Word Form Tags")
    query = f"INSERT INTO tags VALUES (({FORM_QUERY}),?,?)"
    for batch in _batch(entries):
        tags: List[Tuple[str, int, Optional[str], int, str, str]] = []
        for entry in batch:
            eid = entry["id"]
            lid = lexidmap.get(eid, lexid)
            if entry.get("lemma"):
                for tag in entry["lemma"].get("tags", []):
                    tags.append((eid, lid, None, 0, tag["text"], tag["category"]))
            for i, form in enumerate(_forms(entry), 1):
                # rank is not valid in FORM_QUERY for external forms
                rank = -1 if _is_external(form) else i
                for tag in form.get("tags", []):
                    tags.append(
                        (eid, lid, form.get("id"), rank, tag["text"], tag["category"])
                    )
        cur.executemany(query, tags)
        progress.update(len(tags))


def _insert_senses(
    entries: Sequence[_AnyEntry],
    synsets: Sequence[_AnySynset],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Senses")
    ssrank = {
        s: i
        for ss in _local_synsets(synsets)
        for i, s in enumerate(ss.get("members", []))
    }
    query = f"""
        INSERT INTO senses
        VALUES (null,
                ?,
                ?,
                ({ENTRY_QUERY}),
                ?,
                ({SYNSET_QUERY}),
                ?,
                ?,
                ?)
    """
    for batch in _batch(entries):
        data = [
            (
                sense["id"],
                lexid,
                entry["id"],
                lexidmap.get(entry["id"], lexid),
                i,
                sense["synset"],
                lexidmap.get(sense["synset"], lexid),
                ssrank.get(sense["id"], DEFAULT_MEMBER_RANK),
                sense.get("lexicalized", True),
                sense["meta"],
            )
            for entry in batch
            for i, sense in enumerate(_local_senses(_senses(entry)))
        ]
        cur.executemany(query, data)
        progress.update(len(data))


def _insert_adjpositions(
    entries: Sequence[_AnyEntry],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
):
    progress.set(status="Sense Adjpositions")
    data = [
        (s["id"], lexidmap.get(s["id"], lexid), s["adjposition"])
        for e in entries
        for s in _local_senses(_senses(e))
        if s.get("adjposition")
    ]
    query = f"INSERT INTO adjpositions VALUES (({SENSE_QUERY}),?)"
    cur.executemany(query, data)


def _insert_counts(
    entries: Sequence[_AnyEntry],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Counts")
    data = [
        (
            lexid,
            sense["id"],
            lexidmap.get(sense["id"], lexid),
            count["value"],
            count["meta"],
        )
        for entry in entries
        for sense in _senses(entry)
        for count in sense.get("counts", [])
    ]
    query = f"INSERT INTO counts VALUES (null,?,({SENSE_QUERY}),?,?)"
    cur.executemany(query, data)
    progress.update(len(data))


def _collect_frames(lexicon: _AnyLexicon) -> List[lmf.SyntacticBehaviour]:
    # WN-LMF 1.0 syntactic behaviours are on lexical entries, and in
    # WN-LMF 1.1 they are at the lexticon level with IDs. This
    # function normalizes the two variants.

    # IDs are not required and frame strings must be unique in a
    # lexicon, so lookup syntactic behaviours by the frame string
    synbhrs: Dict[str, lmf.SyntacticBehaviour] = {
        frame["subcategorizationFrame"]: {
            "id": frame["id"],
            "subcategorizationFrame": frame["subcategorizationFrame"],
            "senses": frame.get("senses", []),
        }
        for frame in lexicon.get("frames", [])
    }
    # all relevant senses are collected into the 'senses' key
    id_senses_map = {sb["id"]: sb["senses"] for sb in synbhrs.values() if sb.get("id")}
    for entry in _entries(lexicon):
        # for WN-LMF 1.1
        for sense in _local_senses(_senses(entry)):
            for sbid in sense.get("subcat", []):
                id_senses_map[sbid].append(sense["id"])
        # for WN-LMF 1.0
        if _is_external(entry) or not entry.get("frames"):
            continue
        entry = cast(lmf.LexicalEntry, entry)
        all_senses = [s["id"] for s in _senses(entry)]
        for frame in entry.get("frames", []):
            subcat_frame = frame["subcategorizationFrame"]
            if subcat_frame not in synbhrs:
                synbhrs[subcat_frame] = {
                    "subcategorizationFrame": subcat_frame,
                    "senses": [],
                }
            senses = frame.get("senses", []) or all_senses
            synbhrs[subcat_frame]["senses"].extend(senses)
    return list(synbhrs.values())


def _insert_syntactic_behaviours(
    synbhrs: Sequence[lmf.SyntacticBehaviour],
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Syntactic Behaviours")

    query = "INSERT INTO syntactic_behaviours VALUES (null,?,?,?)"
    sbdata = [
        (sb.get("id") or None, lexid, sb["subcategorizationFrame"]) for sb in synbhrs
    ]
    cur.executemany(query, sbdata)

    # syntactic behaviours don't have a required ID; index on frame
    framemap: Dict[str, List[str]] = {
        sb["subcategorizationFrame"]: sb.get("senses", []) for sb in synbhrs
    }
    query = f"""
        INSERT INTO syntactic_behaviour_senses
        VALUES ((SELECT rowid
                   FROM syntactic_behaviours
                  WHERE lexicon_rowid=? AND frame=?),
                ({SENSE_QUERY}))
    """
    sbsdata = [
        (lexid, frame, sid, lexidmap.get(sid, lexid))
        for frame in framemap
        for sid in framemap[frame]
    ]
    cur.executemany(query, sbsdata)

    progress.update(len(synbhrs))


def _insert_sense_relations(
    lexicon: _AnyLexicon,
    lexid: int,
    lexidmap: _LexIdMap,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Sense Relations")
    # need to separate relations into those targeting senses vs synsets
    synset_ids = {ss["id"] for ss in _synsets(lexicon)}
    sense_ids = {s["id"] for e in _entries(lexicon) for s in _senses(e)}
    s_s_rels = []
    s_ss_rels = []
    for entry in _entries(lexicon):
        for sense in _senses(entry):
            slid = lexidmap.get(sense["id"], lexid)
            for relation in sense.get("relations", []):
                target_id = relation["target"]
                tlid = lexidmap.get(target_id, lexid)
                if target_id in sense_ids:
                    s_s_rels.append((sense["id"], slid, tlid, relation))
                elif target_id in synset_ids:
                    s_ss_rels.append((sense["id"], slid, tlid, relation))
                else:
                    raise wn.Error(
                        f"relation target is not a known sense or synset: {target_id}"
                    )
    hyperparams = [
        ("sense_relations", SENSE_QUERY, s_s_rels),
        ("sense_synset_relations", SYNSET_QUERY, s_ss_rels),
    ]
    for table, target_query, rels in hyperparams:
        query = f"""
            INSERT INTO {table}
            VALUES (null,?,({SENSE_QUERY}),({target_query}),({RELTYPE_QUERY}),?)
        """
        for batch in _batch(rels):
            data = [
                (
                    lexid,
                    sense_id,
                    slid,
                    relation["target"],
                    tlid,
                    relation["relType"],
                    relation["meta"],
                )
                for sense_id, slid, tlid, relation in batch
            ]
            cur.executemany(query, data)
            progress.update(len(data))


def _insert_examples(
    objs: Sequence[Union[lmf.Sense, lmf.ExternalSense, lmf.Synset, lmf.ExternalSynset]],
    lexid: int,
    lexidmap: _LexIdMap,
    table: str,
    cur: sqlite3.Cursor,
    progress: ProgressHandler,
) -> None:
    progress.set(status="Examples")
    if table == "sense_examples":
        query = f"INSERT INTO {table} VALUES (null,?,({SENSE_QUERY}),?,?,?)"
    else:
        query = f"INSERT INTO {table} VALUES (null,?,({SYNSET_QUERY}),?,?,?)"
    for batch in _batch(objs):
        data = [
            (
                lexid,
                obj["id"],
                lexidmap.get(obj["id"], lexid),
                example["text"],
                example.get("language"),
                example["meta"],
            )
            for obj in batch
            for example in obj.get("examples", [])
        ]
        # be careful of SQL injection here
        cur.executemany(query, data)
        progress.update(len(data))


def _add_ili(
    source: Path,
    progress: ProgressHandler,
) -> None:
    query = f"""
        INSERT INTO ilis
        VALUES (null,?,({ILISTAT_QUERY}),?,null)
            ON CONFLICT(id) DO
               UPDATE SET status_rowid=excluded.status_rowid,
                          definition=excluded.definition
    """
    with connect() as conn:
        cur = conn.cursor()

        progress.flash(f"Reading ILI file: {source!s}")
        ili = list(_ili.load(source))

        progress.flash("Updating ILI Status Names")
        statuses = set(info.get("status", "active") for info in ili)
        cur.executemany(
            "INSERT OR IGNORE INTO ili_statuses VALUES (null,?)",
            [(stat,) for stat in sorted(statuses)],
        )

        progress.set(count=0, total=len(ili), status="ILI")
        for batch in _batch(ili):
            data = [
                (info["ili"], info.get("status", "active"), info.get("definition"))
                for info in batch
            ]
            cur.executemany(query, data)
            progress.update(len(data))


def remove(
    lexicon: str, progress_handler: Optional[Type[ProgressHandler]] = ProgressBar
) -> None:
    """Remove lexicon(s) from the database.

    The *lexicon* argument is a :ref:`lexicon specifier
    <lexicon-specifiers>`. Note that this removes a lexicon and not a
    project, so the lexicons of projects containing multiple lexicons
    will need to be removed individually or, if applicable, a star
    specifier.

    The *progress_handler* parameter takes a subclass of
    :class:`wn.util.ProgressHandler`. An instance of the class will be
    created, used, and closed by this function.

    >>> wn.remove('ewn:2019')  # removes a single lexicon
    >>> wn.remove('*:1.3+omw')  # removes all lexicons with version 1.3+omw

    """
    if progress_handler is None:
        progress_handler = ProgressHandler
    progress = progress_handler(message="Removing", unit="\be5 operations")

    conn = connect()
    conn.set_progress_handler(progress.update, 100000)
    try:
        for rowid, id, _, _, _, _, version, *_ in find_lexicons(lexicon=lexicon):
            extensions = _find_all_extensions(rowid)

            with conn:

                for ext_id, ext_spec in reversed(extensions):
                    progress.set(status=f"{ext_spec} (extension)")
                    conn.execute("DELETE from lexicons WHERE rowid = ?", (ext_id,))
                    progress.flash(f"Removed {ext_spec}\n")

                spec = f"{id}:{version}"
                extra = f" (and {len(extensions)} extension(s))" if extensions else ""
                progress.set(status=f"{spec}", count=0)
                conn.execute("DELETE from lexicons WHERE rowid = ?", (rowid,))
                progress.flash(f"Removed {spec}{extra}\n")

    finally:
        progress.close()
        conn.set_progress_handler(None, 0)


def _find_all_extensions(rowid: int) -> List[Tuple[int, str]]:
    exts: List[Tuple[int, str]] = []
    for ext_id in get_lexicon_extensions(rowid):
        lexinfo = get_lexicon(ext_id)
        exts.append((ext_id, f"{lexinfo[1]}:{lexinfo[6]}"))
    return exts


def _entries(lex: _AnyLexicon) -> Sequence[_AnyEntry]:
    return lex.get("entries", [])


def _forms(e: _AnyEntry) -> Sequence[_AnyForm]:
    return e.get("forms", [])


def _senses(e: _AnyEntry) -> Sequence[_AnySense]:
    return e.get("senses", [])


def _synsets(lex: _AnyLexicon) -> Sequence[_AnySynset]:
    return lex.get("synsets", [])


def _is_external(
    x: Union[_AnyForm, _AnyLemma, _AnyEntry, _AnySense, _AnySynset]
) -> bool:
    return x.get("external", False) is True


def _local_synsets(synsets: Sequence[_AnySynset]) -> Iterator[lmf.Synset]:
    for ss in synsets:
        if _is_external(ss):
            continue
        yield cast(lmf.Synset, ss)


def _local_entries(entries: Sequence[_AnyEntry]) -> Iterator[lmf.LexicalEntry]:
    for e in entries:
        if _is_external(e):
            continue
        yield cast(lmf.LexicalEntry, e)


def _local_senses(senses: Sequence[_AnySense]) -> Iterator[lmf.Sense]:
    for s in senses:
        if _is_external(s):
            continue
        yield cast(lmf.Sense, s)
