"""Generate structured data from a rendered web page."""

from __future__ import annotations

import asyncio
import hashlib
import json
import logging
import re
import shutil
import subprocess
import time
from dataclasses import dataclass, field
from importlib import resources
from pathlib import Path
from typing import Any
from urllib.parse import urlparse

import wordlift_client
from wordlift_client import ApiClient, Configuration
from wordlift_client import AgentApi
from wordlift_client.models.ask_request import AskRequest
from rdflib import Graph, Namespace, RDF
from rdflib.term import BNode, Identifier, Literal, URIRef

from worai.core.prune_entities_outside_dataset import DEFAULT_BASE_URL
from worai.core.validate_shacl import ValidationResult, validate_file
from worai.errors import ExternalServiceError, UsageError
from worai.seocheck.browser import Browser


_SCHEMA_BASE = "https://schema.org"
_SCHEMA_HTTP = "http://schema.org/"
_AGENT_BASE_URL = "https://api.wordlift.io/agent"
_AGENT_MODEL = "gpt-5.1"
_RR = Namespace("http://www.w3.org/ns/r2rml#")
_RML = Namespace("http://w3id.org/rml/")
_RML_LEGACY = Namespace("http://semweb.mmlab.be/ns/rml#")
_QL = Namespace("http://semweb.mmlab.be/ns/ql#")
_SH = Namespace("http://www.w3.org/ns/shacl#")
_REVIEW_OPTIONAL_EXTRAS = {
    "description",
    "positiveNotes",
    "negativeNotes",
    "reviewBody",
    "image",
    "inLanguage",
    "publisher",
    "datePublished",
}


@dataclass
class RenderOptions:
    url: str
    headless: bool = True
    timeout_ms: int = 30000
    wait_until: str = "networkidle"
    locale: str = "en-US"
    user_agent: str | None = (
        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
    )
    viewport_width: int = 1365
    viewport_height: int = 768
    ignore_https_errors: bool = False


@dataclass
class RenderedPage:
    html: str
    xhtml: str
    status_code: int | None = None
    resources: list[dict] = field(default_factory=list)


@dataclass
class CleanupOptions:
    max_xhtml_chars: int = 40000
    max_text_node_chars: int = 400
    remove_tags: tuple[str, ...] = (
        "script",
        "style",
        "noscript",
        "svg",
        "canvas",
        "iframe",
        "form",
        "input",
        "button",
        "nav",
        "aside",
    )


@dataclass
class StructuredDataOptions:
    url: str
    target_type: str | None
    dataset_uri: str
    headless: bool = True
    timeout_ms: int = 30000
    wait_until: str = "networkidle"
    max_retries: int = 2
    max_xhtml_chars: int = 40000
    max_text_node_chars: int = 400
    max_nesting_depth: int = 2
    verbose: bool = True


@dataclass
class StructuredDataResult:
    jsonld: dict[str, Any]
    yarrml: str
    jsonld_filename: str
    yarrml_filename: str


def _build_client(api_key: str, base_url: str) -> ApiClient:
    config = Configuration(host=base_url)
    config.api_key["ApiKey"] = api_key
    config.api_key_prefix["ApiKey"] = "Key"
    return ApiClient(config)


def _build_agent_client(api_key: str) -> ApiClient:
    config = Configuration(host=_AGENT_BASE_URL)
    config.api_key["ApiKey"] = api_key
    config.api_key_prefix["ApiKey"] = "Key"
    return ApiClient(config)


async def get_dataset_uri_async(api_key: str, base_url: str = DEFAULT_BASE_URL) -> str:
    async with _build_client(api_key, base_url) as api_client:
        api = wordlift_client.AccountApi(api_client)
        account = await api.get_me()
    dataset_uri = getattr(account, "dataset_uri", None)
    if not dataset_uri:
        raise RuntimeError("Failed to resolve dataset_uri from account get_me.")
    return dataset_uri


def get_dataset_uri(api_key: str, base_url: str = DEFAULT_BASE_URL) -> str:
    return asyncio.run(get_dataset_uri_async(api_key, base_url))


def render_html(options: RenderOptions) -> RenderedPage:
    ignore_https_errors = options.ignore_https_errors or _is_localhost_url(options.url)
    with Browser(
        headless=options.headless,
        timeout_ms=options.timeout_ms,
        wait_until=options.wait_until,
        locale=options.locale,
        user_agent=options.user_agent,
        viewport_width=options.viewport_width,
        viewport_height=options.viewport_height,
        ignore_https_errors=ignore_https_errors,
    ) as browser:
        page, response, _elapsed_ms, resources = browser.open(options.url)
        if page is None:
            raise RuntimeError("Failed to open page in browser.")
        try:
            html = _safe_page_content(page, options.timeout_ms)
        finally:
            page.close()
    xhtml = _html_to_xhtml(html)
    status_code = None
    if response is not None:
        try:
            status_code = response.status
        except Exception:
            status_code = None
    return RenderedPage(html=html, xhtml=xhtml, status_code=status_code, resources=resources)


def _is_localhost_url(url: str) -> bool:
    try:
        parsed = urlparse(url)
    except Exception:
        return False
    host = (parsed.hostname or "").lower()
    return host == "localhost" or host.endswith(".localhost")


def _html_to_xhtml(html: str) -> str:
    html = re.sub(r"<!DOCTYPE[^>]*>", "", html, flags=re.IGNORECASE)
    html = _strip_invalid_xml_chars(html)
    try:
        from lxml import html as lxml_html
    except Exception as exc:
        raise UsageError("lxml is required for XHTML output. Install with: pip install lxml") from exc
    try:
        parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
        doc = lxml_html.document_fromstring(html, parser=parser)
        _sanitize_xhtml_tree(doc)
        xhtml = lxml_html.tostring(doc, encoding="unicode", method="xml")
        return _strip_invalid_xml_chars(xhtml)
    except Exception as exc:
        raise UsageError("Failed to convert HTML to XHTML.") from exc


def _safe_page_content(page: Any, timeout_ms: int, retries: int = 3) -> str:
    for attempt in range(retries + 1):
        try:
            return page.content()
        except Exception:
            if attempt >= retries:
                raise
            try:
                page.wait_for_load_state("networkidle", timeout=timeout_ms)
            except Exception:
                try:
                    page.wait_for_load_state("load", timeout=timeout_ms)
                except Exception:
                    pass
            time.sleep(0.2)
    return page.content()


def clean_xhtml(xhtml: str, options: CleanupOptions) -> str:
    try:
        from lxml import html as lxml_html
    except Exception as exc:
        raise UsageError("lxml is required for XHTML cleanup. Install with: pip install lxml") from exc
    parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
    doc = lxml_html.document_fromstring(xhtml, parser=parser)
    _strip_unwanted_tags(doc, options.remove_tags)
    _compact_text_nodes(doc, options.max_text_node_chars)
    _cap_text_content(doc, options.max_xhtml_chars)
    cleaned = lxml_html.tostring(doc, encoding="unicode", method="xml")
    if len(cleaned) > options.max_xhtml_chars:
        _trim_elements_to_size(doc, options.max_xhtml_chars)
        cleaned = lxml_html.tostring(doc, encoding="unicode", method="xml")
    return cleaned


def normalize_type(value: str) -> str:
    value = value.strip()
    if value.startswith("schema:"):
        return value.split(":", 1)[1]
    if value.startswith("http://schema.org/"):
        return value.split("/", 3)[-1]
    if value.startswith("https://schema.org/"):
        return value.split("/", 3)[-1]
    return value


_GOOGLE_SHAPES_CACHE: Graph | None = None
_SCHEMA_SHAPES_CACHE: Graph | None = None
_SCHEMA_PROP_CACHE: set[str] | None = None
_SCHEMA_RANGE_CACHE: dict[str, dict[str, set[str]]] | None = None


def _load_google_shapes() -> Graph:
    global _GOOGLE_SHAPES_CACHE
    if _GOOGLE_SHAPES_CACHE is not None:
        return _GOOGLE_SHAPES_CACHE
    graph = Graph()
    shapes_dir = resources.files("worai.validate.shacls")
    for entry in shapes_dir.iterdir():
        if not entry.is_file() or not entry.name.endswith(".ttl"):
            continue
        if entry.name.startswith("google-") or entry.name == "review-snippet.ttl":
            graph.parse(data=entry.read_text(encoding="utf-8"), format="turtle")
    _GOOGLE_SHAPES_CACHE = graph
    return graph


def _load_schema_shapes() -> Graph:
    global _SCHEMA_SHAPES_CACHE
    if _SCHEMA_SHAPES_CACHE is not None:
        return _SCHEMA_SHAPES_CACHE
    graph = Graph()
    shapes_dir = resources.files("worai.validate.shacls")
    schema_path = shapes_dir.joinpath("schemaorg-grammar.ttl")
    if not schema_path.is_file():
        raise UsageError("schemaorg-grammar.ttl not found. Regenerate with scripts/generate_schema_shacls.py.")
    graph.parse(data=schema_path.read_text(encoding="utf-8"), format="turtle")
    _SCHEMA_SHAPES_CACHE = graph
    return graph


def _schema_property_set() -> set[str]:
    global _SCHEMA_PROP_CACHE
    if _SCHEMA_PROP_CACHE is not None:
        return _SCHEMA_PROP_CACHE
    graph = _load_schema_shapes()
    props: set[str] = set()
    for prop in graph.objects(None, _SH.path):
        name = _path_to_string(graph, prop)
        if not name:
            continue
        props.add(name)
        props.add(name.split(".", 1)[0])
    _SCHEMA_PROP_CACHE = props
    return props


def _rdf_list_items(graph: Graph, head: Identifier) -> list[Identifier]:
    items: list[Identifier] = []
    current: Identifier | None = head
    while current and current != RDF.nil:
        first = graph.value(current, RDF.first)
        if first is None:
            break
        items.append(first)
        current = graph.value(current, RDF.rest)
    return items


def _schema_property_ranges() -> dict[str, dict[str, set[str]]]:
    global _SCHEMA_RANGE_CACHE
    if _SCHEMA_RANGE_CACHE is not None:
        return _SCHEMA_RANGE_CACHE
    graph = _load_schema_shapes()
    ranges: dict[str, dict[str, set[str]]] = {}
    for shape in graph.subjects(_SH.targetClass, None):
        target_class = graph.value(shape, _SH.targetClass)
        type_name = _short_schema_name(target_class)
        if not type_name:
            continue
        for prop in graph.objects(shape, _SH.property):
            path = graph.value(prop, _SH.path)
            if path is None:
                continue
            prop_name = _path_to_string(graph, path)
            if not prop_name:
                continue
            or_list = graph.value(prop, _SH["or"])
            if or_list is None:
                continue
            for item in _rdf_list_items(graph, or_list):
                class_node = graph.value(item, _SH["class"])
                class_name = _short_schema_name(class_node)
                if not class_name:
                    continue
                ranges.setdefault(type_name, {}).setdefault(prop_name, set()).add(class_name)
    _SCHEMA_RANGE_CACHE = ranges
    return ranges


def _short_schema_name(value: Identifier) -> str | None:
    if not isinstance(value, URIRef):
        return None
    text = str(value)
    if text.startswith(_SCHEMA_BASE):
        return text[len(_SCHEMA_BASE) + 1 :]
    if text.startswith(_SCHEMA_HTTP):
        return text[len(_SCHEMA_HTTP) :]
    return None


def _path_to_string(graph: Graph, path: Identifier) -> str | None:
    if isinstance(path, URIRef):
        return _short_schema_name(path)
    if isinstance(path, BNode):
        parts: list[str] = []
        current: Identifier | None = path
        while current and current != RDF.nil:
            first = graph.value(current, RDF.first)
            if first is None:
                break
            name = _short_schema_name(first)
            if not name:
                break
            parts.append(name)
            current = graph.value(current, RDF.rest)
        if parts:
            return ".".join(parts)
    return None


def _property_guide_for_type(type_name: str) -> dict[str, list[str]]:
    type_name = normalize_type(type_name)
    targets = {URIRef(f"{_SCHEMA_BASE}/{type_name}"), URIRef(f"{_SCHEMA_HTTP}{type_name}")}

    required: set[str] = set()
    recommended: set[str] = set()
    google_graph = _load_google_shapes()
    for target in targets:
        for shape in google_graph.subjects(_SH.targetClass, target):
            for prop in google_graph.objects(shape, _SH.property):
                path = google_graph.value(prop, _SH.path)
                if path is None:
                    continue
                min_count = google_graph.value(prop, _SH.minCount)
                if isinstance(min_count, Literal):
                    try:
                        if int(min_count) < 1:
                            continue
                    except Exception:
                        continue
                else:
                    continue
                prop_name = _path_to_string(google_graph, path)
                if not prop_name:
                    continue
                severity = google_graph.value(prop, _SH.severity)
                if severity == _SH.Warning:
                    recommended.add(prop_name)
                else:
                    required.add(prop_name)

    schema_props: set[str] = set()
    schema_graph = _load_schema_shapes()
    for target in targets:
        for shape in schema_graph.subjects(_SH.targetClass, target):
            for prop in schema_graph.objects(shape, _SH.property):
                path = schema_graph.value(prop, _SH.path)
                if path is None:
                    continue
                prop_name = _path_to_string(schema_graph, path)
                if not prop_name:
                    continue
                schema_props.add(prop_name)

    optional = sorted(schema_props.difference(required).difference(recommended))

    return {
        "required": sorted(required),
        "recommended": sorted(recommended),
        "optional": optional,
        "schema": sorted(schema_props),
    }


def _related_types_for_type(
    type_name: str,
    property_guide: dict[str, list[str]],
    ranges: dict[str, dict[str, set[str]]],
) -> list[str]:
    related: set[str] = set()
    prop_ranges = ranges.get(type_name, {})
    prop_candidates = property_guide.get("required", []) + property_guide.get("recommended", [])
    if not prop_candidates:
        prop_candidates = property_guide.get("schema", [])
    for prop in prop_candidates:
        base = prop.split(".", 1)[0]
        for range_type in prop_ranges.get(base, set()):
            if range_type == "Thing":
                continue
            related.add(range_type)
    return sorted(related)


def property_guides_with_related(
    type_name: str,
    max_depth: int = 2,
) -> dict[str, dict[str, list[str]]]:
    type_name = normalize_type(type_name)
    ranges = _schema_property_ranges()
    guides: dict[str, dict[str, list[str]]] = {}
    queue: list[tuple[str, int]] = [(type_name, 0)]
    seen: set[str] = set()

    while queue:
        current, depth = queue.pop(0)
        if current in seen:
            continue
        seen.add(current)
        guide = _property_guide_for_type(current)
        guides[current] = guide
        if depth >= max_depth:
            continue
        for related in _related_types_for_type(current, guide, ranges):
            if related not in seen:
                queue.append((related, depth + 1))

    return guides


def shape_specs_for_type(type_name: str) -> list[str]:
    return all_shape_specs()


def shape_specs_for_types(type_names: list[str]) -> list[str]:
    return all_shape_specs()


def all_shape_specs() -> list[str]:
    shapes_dir = resources.files("worai.validate.shacls")
    shape_specs: list[str] = []
    for entry in shapes_dir.iterdir():
        if not entry.is_file() or not entry.name.endswith(".ttl"):
            continue
        if entry.name not in shape_specs:
            shape_specs.append(entry.name)
    if "schemaorg-grammar.ttl" not in shape_specs:
        shape_specs.append("schemaorg-grammar.ttl")
    return shape_specs


_INVALID_XML_CHARS_RE = re.compile(
    r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]"
)

_XML_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.:-]*$")


def _strip_invalid_xml_chars(value: str) -> str:
    return _INVALID_XML_CHARS_RE.sub("", value)


def _sanitize_xhtml_tree(doc: Any) -> None:
    for element in doc.iter():
        if not hasattr(element, "attrib"):
            continue
        for attr in list(element.attrib):
            if not _XML_NAME_RE.match(attr):
                del element.attrib[attr]
                continue
            value = element.attrib.get(attr)
            if isinstance(value, str):
                element.attrib[attr] = _strip_invalid_xml_chars(value)


def _strip_unwanted_tags(doc: Any, tags: tuple[str, ...]) -> None:
    if not tags:
        return
    tag_expr = " | ".join(f"//{tag}" for tag in tags)
    if not tag_expr:
        return
    for element in doc.xpath(tag_expr):
        parent = element.getparent()
        if parent is not None:
            parent.remove(element)


def _compact_text(value: str | None, max_chars: int) -> str | None:
    if value is None:
        return None
    text = re.sub(r"\s+", " ", value).strip()
    if not text:
        return None
    if max_chars > 0 and len(text) > max_chars:
        if max_chars <= 3:
            text = text[:max_chars]
        else:
            text = text[: max_chars - 3].rstrip() + "..."
    return text


def _compact_text_nodes(doc: Any, max_chars: int) -> None:
    for element in doc.iter():
        if hasattr(element, "text"):
            element.text = _compact_text(element.text, max_chars)
        if hasattr(element, "tail"):
            element.tail = _compact_text(element.tail, max_chars)


def _cap_text_content(doc: Any, max_chars: int) -> None:
    if max_chars <= 0:
        return
    remaining = max_chars
    for element in doc.iter():
        if hasattr(element, "text") and element.text:
            if len(element.text) <= remaining:
                remaining -= len(element.text)
            else:
                element.text = element.text[: max(0, remaining)].rstrip()
                remaining = 0
        if remaining <= 0:
            _clear_text_after(doc, element)
            break
        if hasattr(element, "tail") and element.tail:
            if len(element.tail) <= remaining:
                remaining -= len(element.tail)
            else:
                element.tail = element.tail[: max(0, remaining)].rstrip()
                remaining = 0
        if remaining <= 0:
            _clear_text_after(doc, element)
            break


def _clear_text_after(doc: Any, stop_element: Any) -> None:
    seen = False
    for element in doc.iter():
        if element is stop_element:
            seen = True
            continue
        if not seen:
            continue
        if hasattr(element, "text") and element.text:
            element.text = None
        if hasattr(element, "tail") and element.tail:
            element.tail = None


def _trim_elements_to_size(doc: Any, max_chars: int) -> None:
    if max_chars <= 0:
        return
    try:
        from lxml import html as lxml_html
    except Exception:
        return
    elements = list(doc.iter())
    for element in reversed(elements):
        parent = element.getparent()
        if parent is None:
            continue
        parent.remove(element)
        current = lxml_html.tostring(doc, encoding="unicode", method="xml")
        if len(current) <= max_chars:
            return


def _slugify(value: str, default: str) -> str:
    cleaned = re.sub(r"[^A-Za-z0-9]+", "-", value.strip().lower()).strip("-")
    return cleaned or default


def _dash_type(value: str) -> str:
    value = re.sub(r"[^A-Za-z0-9]+", "-", value.strip())
    value = re.sub(r"(?<!^)(?=[A-Z])", "-", value)
    return re.sub(r"-+", "-", value).strip("-").lower()


def _pluralize(value: str) -> str:
    if value.endswith("y") and len(value) > 1 and value[-2] not in "aeiou":
        return value[:-1] + "ies"
    if value.endswith(("s", "x", "z", "ch", "sh")):
        return value + "es"
    return value + "s"


def _hash_url(url: str, length: int = 12) -> str:
    return hashlib.sha256(url.encode("utf-8")).hexdigest()[:length]


def build_output_basename(url: str, default: str = "page") -> str:
    parsed = urlparse(url)
    base = f"{parsed.netloc}{parsed.path}".strip("/")
    slug = _slugify(base or url, default=default)
    return f"{slug}--{_hash_url(url)}"

def build_id(
    dataset_uri: str,
    type_name: str,
    name: str,
    url: str | None,
    index: int,
) -> str:
    return build_id_base(dataset_uri, type_name, name, url, index)


def build_id_base(
    base_uri: str,
    type_name: str,
    name: str,
    url: str | None,
    index: int,
) -> str:
    base = base_uri.rstrip("/")
    dashed_type = _dash_type(type_name)
    plural_type = _pluralize(dashed_type)
    name_slug = _slugify(name, default=_dash_type(type_name))
    if url:
        suffix = _hash_url(url)
    else:
        suffix = str(index)
    return f"{base}/{plural_type}/{name_slug}-{suffix}"


def _format_prop_list(items: list[str]) -> str:
    if not items:
        return "none"
    return ", ".join(items)


def _agent_prompt(
    url: str,
    html: str,
    target_type: str | None,
    property_guides: dict[str, dict[str, list[str]]] | None = None,
    missing_required: list[str] | None = None,
    missing_recommended: list[str] | None = None,
    previous_yarrml: str | None = None,
    validation_errors: list[str] | None = None,
    validation_report: list[str] | None = None,
    xpath_warnings: list[str] | None = None,
    allow_properties: dict[str, list[str]] | None = None,
    quality_feedback: list[str] | None = None,
) -> str:
    target = target_type or "AUTO"
    guide_lines: list[str] = []
    if property_guides:
        guide_lines.append("Property guide by type (Google required/recommended + Schema.org grammar):")
        for type_name, guide in property_guides.items():
            guide_lines.append(f"- {type_name}:")
            guide_lines.append(f"  - Required (Google): {_format_prop_list(guide.get('required', []))}")
            guide_lines.append(f"  - Recommended (Google): {_format_prop_list(guide.get('recommended', []))}")
            guide_lines.append(
                "  - Optional (Schema.org, excluding required/recommended): "
                f"{_format_prop_list(guide.get('optional', []))}"
            )
            guide_lines.append(
                f"  - All Schema.org properties for {type_name}: {_format_prop_list(guide.get('schema', []))}"
            )
        guide_lines.append("")
        guide_lines.append(
            "If a required property is not present on the page, omit it (do not fabricate)."
        )
        guide_lines.append("Only use properties listed in the guide for each type.")
        guide_lines.append("")
    if allow_properties:
        guide_lines.append("Allowed properties (Google only):")
        for type_name, props in allow_properties.items():
            guide_lines.append(f"- {type_name}: {_format_prop_list(props)}")
        guide_lines.append("")

    refine_lines: list[str] = []
    if missing_required:
        refine_lines.append("Missing required properties in the previous mapping:")
        refine_lines.append(", ".join(missing_required))
        refine_lines.append(
            "Update the mapping to add these properties if the data exists on the page."
        )
        refine_lines.append("Keep existing correct mappings and selectors.")
        refine_lines.append("")
    if missing_recommended:
        refine_lines.append("Missing recommended properties in the previous mapping:")
        refine_lines.append(", ".join(missing_recommended))
        refine_lines.append(
            "Add these properties if the data exists on the page."
        )
        refine_lines.append("")
    if validation_errors:
        refine_lines.append("Validation errors from the previous mapping:")
        refine_lines.extend(validation_errors)
        refine_lines.append("Fix these issues without fabricating data.")
        refine_lines.append("")
    if validation_report:
        refine_lines.append("Validation report from the previous mapping:")
        refine_lines.extend(validation_report)
        refine_lines.append("Use the report to fix the mapping without fabricating data.")
        refine_lines.append("")
    if xpath_warnings:
        refine_lines.append("XPath evaluation warnings from the previous mapping:")
        refine_lines.extend(xpath_warnings)
        refine_lines.append("Fix the XPath selectors that returned no results.")
        refine_lines.append("")
    if quality_feedback:
        refine_lines.append("Quality feedback from the previous mapping:")
        refine_lines.extend(quality_feedback)
        refine_lines.append("Improve the mapping to raise the quality score while only using data present in XHTML.")
        refine_lines.append("")
    if previous_yarrml:
        refine_lines.append("Previous mapping:")
        refine_lines.append(previous_yarrml.strip())
        refine_lines.append("")

    guide_text = "\n".join(guide_lines) if guide_lines else ""
    refine_text = "\n".join(refine_lines) if refine_lines else ""
    return (
        f"analyze the entities on this webpage: {url}\n"
        "\n"
        "You are a structured data extraction agent.\n"
        "Goal: produce a YARRRML mapping using XPath only.\n"
        "Use the provided XHTML source instead of fetching the URL.\n"
        "Do NOT parse any existing structured data (JSON-LD, RDFa, Microdata).\n"
        "Do NOT emit @id values. IDs will be assigned locally.\n"
        "Output ONLY the YARRRML mapping (no prose, no code fences).\n"
        "\n"
        f"Target Schema.org type: {target}\n"
        "\n"
        "Requirements:\n"
        "- Use XPath in all selectors.\n"
        "- Use $(xpath) for XPath references (not {xpath}).\n"
        "- Do NOT wrap XPath expressions in quotes inside $(...). Use $(/path), not $(\"/path\").\n"
        "- Always quote attribute values in XPath (e.g., @id=\"...\"). Do NOT use @id=foo.\n"
        "- The main mapping must include schema:url with the exact URL.\n"
        "- Always include schema:name for every mapped node.\n"
        "- Include schema:description for Review if available.\n"
        "- Include schema:image if available (prefer og:image). \n"
        "- Include schema:inLanguage if available (html/@lang). \n"
        "- Include schema:publisher if available (prefer og:site_name as Organization). \n"
        "- Include schema:reviewBody for Review if available (main article text). Prefer the paragraph immediately following the H1\n"
        "  (e.g., following-sibling::p[1]) and only use class-based selectors if necessary.\n"
        "- Include schema:datePublished for Review if available (time/@datetime or meta[property=\"article:published_time\"],\n"
        "  otherwise use the first byline date).\n"
        "- Include positiveNotes/negativeNotes for Review if available.\n"
        "- Include relevant properties for the main type.\n"
        "- If Target Schema.org type is AUTO, infer the best type and use it.\n"
        "- Define dependent nodes as separate mappings and link them from the main mapping.\n"
        "- Prefer reusable XPath selectors that generalize across pages using the same template.\n"
        "- Avoid brittle selectors that depend on full class names, IDs, or numeric suffixes unless there is no alternative.\n"
        "- Prefer structural paths (head/meta, main/h1, time[@datetime], link[@rel], figure/img) and stable attributes.\n"
        "- If you must use classes or IDs, prefer contains(@class, 'stable-token') over exact matches and avoid numeric IDs.\n"
        "- NEVER use table IDs with numeric suffixes (e.g., tablepress-12345). Instead, locate tables by header text\n"
        "  (e.g., th contains 'APR'/'rating') and then select the adjacent cell by position or data-th.\n"
        "- Do NOT key selectors off a specific person name or URL slug; use byline labels like \"Written by\" or metadata instead.\n"
        "- For author, prefer metadata or rel links first (meta[name=author], meta[property=article:author], link[rel=author]) before class-based selectors.\n"
        "- If the page shows a byline label (e.g., \"Written by\"), select the author link or text immediately following that label.\n"
        "- For positiveNotes/negativeNotes (Review/Product and subclasses only), anchor on semantic headings (Pros/Cons, Advantages/Disadvantages,\n"
        "  What we like/What we don't like). Prefer heading text matches (contains(., 'Pros')) over IDs/classes,\n"
        "  and select the closest following list items (li) from ul/ol, rows from tables, or terms/defs from dl.\n"
        "  Detect the page language and include localized heading variants with English as a fallback. Avoid site-specific classes/IDs unless there is no alternative.\n"
        "- Only include reviewRating if the page explicitly provides a rating score (stars or numeric rating). Do NOT infer ratings from APR/fee tables or unrelated metrics.\n"
        "- Do NOT use hard-coded literal values. All values must come from XPath except schema:url.\n"
        "- ratingValue must be a literal extracted from XPath (not an IRI).\n"
        "- reviewRating must point to a Rating node.\n"
        "- author must be a Person or Organization node.\n"
        "\n"
        f"{guide_text}"
        f"{refine_text}"
        "XHTML:\n"
        f"{html}\n"
    )


def _quality_prompt(
    url: str,
    xhtml: str,
    jsonld: dict[str, Any] | list[Any],
    property_guides: dict[str, dict[str, list[str]]] | None,
    target_type: str | None,
) -> str:
    guide_lines: list[str] = []
    if property_guides:
        guide_lines.append("Property guide by type (Google required/recommended + Schema.org grammar):")
        for type_name, guide in property_guides.items():
            guide_lines.append(f"- {type_name}:")
            guide_lines.append(f"  - Required (Google): {_format_prop_list(guide.get('required', []))}")
            guide_lines.append(f"  - Recommended (Google): {_format_prop_list(guide.get('recommended', []))}")
            guide_lines.append(
                "  - Optional (Schema.org, excluding required/recommended): "
                f"{_format_prop_list(guide.get('optional', []))}"
            )
        guide_lines.append("")
    guide_text = "\n".join(guide_lines) if guide_lines else ""
    payload = json.dumps(jsonld, ensure_ascii=True)
    return (
        f"analyze the entities on this webpage: {url}\n"
        "\n"
        "You are evaluating structured data quality.\n"
        "Compare the XHTML and JSON-LD. Only count properties that are present in XHTML.\n"
        "Do NOT penalize missing properties if they do not appear in the XHTML.\n"
        "Return a JSON object with keys: score (0-10 integer), missing_in_jsonld (list),\n"
        "suggested_xpath (object mapping property -> XPath), notes (list).\n"
        "Use XPath in suggested_xpath and keep it generic/reusable.\n"
        "\n"
        f"Target Schema.org type: {target_type or 'AUTO'}\n"
        "\n"
        f"{guide_text}"
        "XHTML:\n"
        f"{xhtml}\n"
        "\n"
        "JSON-LD:\n"
        f"{payload}\n"
    )


async def _ask_agent_async(prompt: str, api_key: str, model: str | None = None) -> object:
    async with _build_agent_client(api_key) as api_client:
        api = AgentApi(api_client)
        ask_request = AskRequest(message=prompt, model=model or _AGENT_MODEL)
        return await api.ask_request_api_ask_post(ask_request)


def _collect_strings(payload: Any, results: list[str]) -> None:
    if isinstance(payload, str):
        if payload.strip():
            results.append(payload)
        return
    if isinstance(payload, dict):
        for value in payload.values():
            _collect_strings(value, results)
        return
    if isinstance(payload, list):
        for value in payload:
            _collect_strings(value, results)


def _extract_agent_text(payload: Any) -> str | None:
    if isinstance(payload, str) and payload.strip():
        return payload.strip()
    if isinstance(payload, dict):
        for key in ("response", "answer", "content", "result", "output", "text", "message"):
            if key in payload:
                value = _extract_agent_text(payload.get(key))
                if value:
                    return value
    strings: list[str] = []
    _collect_strings(payload, strings)
    for value in strings:
        if "mappings:" in value or "prefixes:" in value:
            return value.strip()
    for value in strings:
        if value.strip():
            return value.strip()
    return None


def _extract_agent_json(payload: Any) -> dict[str, Any] | None:
    if isinstance(payload, dict):
        return payload
    text = _extract_agent_text(payload)
    if not text:
        return None
    start = text.find("{")
    end = text.rfind("}")
    if start == -1 or end == -1 or end <= start:
        return None
    snippet = text[start : end + 1]
    try:
        return json.loads(snippet)
    except Exception:
        return None


def ask_agent_for_yarrml(
    api_key: str,
    url: str,
    html: str,
    target_type: str | None,
    debug: bool = False,
    debug_path: Path | None = None,
    property_guides: dict[str, dict[str, list[str]]] | None = None,
    missing_required: list[str] | None = None,
    missing_recommended: list[str] | None = None,
    previous_yarrml: str | None = None,
    validation_errors: list[str] | None = None,
    validation_report: list[str] | None = None,
    xpath_warnings: list[str] | None = None,
    allow_properties: dict[str, list[str]] | None = None,
    quality_feedback: list[str] | None = None,
) -> str:
    prompt = _agent_prompt(
        url,
        html,
        target_type,
        property_guides=property_guides,
        missing_required=missing_required,
        missing_recommended=missing_recommended,
        previous_yarrml=previous_yarrml,
        validation_errors=validation_errors,
        validation_report=validation_report,
        xpath_warnings=xpath_warnings,
        allow_properties=allow_properties,
        quality_feedback=quality_feedback,
    )
    try:
        response = asyncio.run(_ask_agent_async(prompt, api_key))
    except Exception as exc:
        raise ExternalServiceError(f"Agent request failed: {exc}") from exc

    if isinstance(response, dict):
        data = response
    else:
        try:
            data = response.model_dump()
        except Exception:
            data = {}

    if debug and debug_path is not None:
        debug_path.parent.mkdir(parents=True, exist_ok=True)
        debug_payload = {
            "prompt": prompt,
            "response": data,
        }
        debug_path.write_text(json.dumps(debug_payload, indent=2))

    extracted = _extract_agent_text(data)
    if extracted:
        return extracted

    raise ExternalServiceError("Agent response did not include YARRRML content.")


def ask_agent_for_quality(
    api_key: str,
    url: str,
    xhtml: str,
    jsonld: dict[str, Any] | list[Any],
    property_guides: dict[str, dict[str, list[str]]] | None,
    target_type: str | None,
) -> dict[str, Any] | None:
    prompt = _quality_prompt(url, xhtml, jsonld, property_guides, target_type)
    try:
        response = asyncio.run(_ask_agent_async(prompt, api_key))
    except Exception as exc:
        raise ExternalServiceError(f"Agent quality request failed: {exc}") from exc
    return _extract_agent_json(response)


def _replace_sources_with_file(yarrml: str, file_uri: str) -> str:
    pattern = re.compile(r"(\[\s*['\"])([^'\"]+)(['\"]\s*,\s*['\"]xpath['\"])")
    inline_pattern = re.compile(r"(\[\s*)([^,\]]+?~xpath)(\s*,)")

    def repl(match: re.Match[str]) -> str:
        return f"{match.group(1)}{file_uri}{match.group(3)}"

    def repl_inline(match: re.Match[str]) -> str:
        return f"{match.group(1)}{file_uri}~xpath{match.group(3)}"

    yarrml = pattern.sub(repl, yarrml)
    return inline_pattern.sub(repl_inline, yarrml)


def _replace_sources_with_placeholder(yarrml: str, placeholder: str) -> str:
    pattern = re.compile(r"(\[\s*['\"])([^'\"]+)(['\"]\s*,\s*['\"]xpath['\"])")
    inline_pattern = re.compile(r"(\[\s*)([^,\]]+?~xpath)(\s*,)")

    def repl(match: re.Match[str]) -> str:
        return f"{match.group(1)}{placeholder}{match.group(3)}"

    def repl_inline(match: re.Match[str]) -> str:
        return f"{match.group(1)}{placeholder}~xpath{match.group(3)}"

    yarrml = pattern.sub(repl, yarrml)
    return inline_pattern.sub(repl_inline, yarrml)


def make_reusable_yarrrml(yarrml: str, url: str, source_placeholder: str = "__XHTML__") -> str:
    normalized = _replace_sources_with_placeholder(yarrml, source_placeholder)
    escaped_url = re.escape(url)
    normalized = re.sub(
        rf"(schema:url\s*,\s*['\"])({escaped_url})(['\"])",
        r"\1__URL__\3",
        normalized,
    )
    return normalized


def _strip_quotes(value: str | None) -> str:
    if not isinstance(value, str):
        return ""
    value = value.strip()
    if (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
        return value[1:-1]
    return value


def _strip_wrapped_list(value: str | None) -> str:
    if not isinstance(value, str):
        return ""
    value = value.strip()
    if value.startswith("[") and value.endswith("]"):
        value = value[1:-1].strip()
    return _strip_quotes(value)


def _strip_all_quotes(value: str | None) -> str:
    if not isinstance(value, str):
        return ""
    value = value.strip()
    while (value.startswith('"') and value.endswith('"')) or (value.startswith("'") and value.endswith("'")):
        value = value[1:-1].strip()
    return value


def _normalize_xpath_literal(value: str) -> str:
    value = value.strip()
    if value.startswith("{") and value.endswith("}"):
        inner = value[1:-1].strip()
        return f"$({inner})"
    if value.startswith("$(xpath)/"):
        tail = value[len("$(xpath)") :]
        return f"$({tail})"
    if value.startswith("$(xpath://") or value.startswith("$(xpath:/"):
        tail = value[len("$(xpath:") :]
        return f"$({tail})"
    if value.startswith("$(") and value.endswith(")"):
        inner = value[2:-1].strip()
        if (inner.startswith('"') and inner.endswith('"')) or (inner.startswith("'") and inner.endswith("'")):
            return f"$({inner[1:-1]})"
    if value.startswith("$(xpath)',"):
        _, _, tail = value.partition(",")
        tail = _strip_all_quotes(tail.strip())
        return _normalize_xpath_literal(f"$({tail})")
    return value


def _looks_like_xpath(value: str) -> bool:
    value = value.strip()
    return (
        (value.startswith("$(") and value.endswith(")"))
        or value.startswith("/")
        or value.startswith(".//")
        or value.startswith("//")
        or value.startswith("normalize-")
        or value.startswith("normalize(")
        or value.startswith("string(")
        or value.startswith("concat(")
    )


def _simplify_xpath(value: str) -> str:
    value = value.strip()
    if value.startswith("$(") and value.endswith(")"):
        value = value[2:-1].strip()
        if value.startswith("xpath="):
            value = value[len("xpath=") :].strip()
        if value.startswith("xpath://") or value.startswith("xpath:/"):
            value = value[len("xpath:") :]
    if value.startswith(('"', "'")) and value.endswith(('"', "'")) and len(value) >= 2:
        value = value[1:-1]
    value = value.replace('\\"', '"').replace("\\'", "'")
    match = re.match(r"(?:normalize-space|string)\((.+)\)$", value)
    if match:
        return match.group(1).strip()
    return value


def _normalize_xpath_reference(value: str) -> str:
    value = value.strip()
    value = value.replace("/text()", "")
    value = value.replace("text()", "")
    value = re.sub(r"contains\(@class,\s*\"([^\"]+)\"\)", r"@class=\"\1\"", value)
    value = re.sub(r"contains\(@class,\s*'([^']+)'\)", r"@class=\"\1\"", value)
    value = re.sub(r"contains\(@id,\s*\"([^\"]+)\"\)", r"@id=\"\1\"", value)
    value = re.sub(r"contains\(@id,\s*'([^']+)'\)", r"@id=\"\1\"", value)
    return value


def _first_list_value(value: str) -> str:
    value = value.strip()
    if value.startswith("[") and value.endswith("]"):
        match = re.search(r"['\"]([^'\"]+)['\"]", value)
        if match:
            return match.group(1)
    return _strip_all_quotes(_strip_wrapped_list(value))


def _normalize_agent_yarrml(
    raw: str,
    url: str,
    file_uri: str,
    target_type: str | None,
) -> tuple[str, list[dict[str, Any]]]:
    raw = _quote_unquoted_xpath_attributes(raw)
    raw = re.sub(r"(['\"])\\{([^{}]+)\\}\\1", r"\\1$(\\2)\\1", raw)
    raw = re.sub(r"(['\"])\\$\\(xpath\\)\\1\\s*,\\s*(['\"])([^'\"]+)\\2", r"\\1$(\\3)\\1", raw)
    lines = raw.splitlines()
    mappings: list[dict[str, Any]] = []
    current: dict[str, Any] | None = None
    in_mappings = False
    mappings_indent: int | None = None
    ignore_keys = {"mappings", "prefixes", "sources", "po"}
    last_p: str | None = None
    pending_o: str | None = None
    in_props_block = False
    props_indent = 0
    in_sources = False
    sources_indent: int | None = None

    for line in lines:
        stripped = line.strip()
        indent = len(line) - len(line.lstrip(" "))
        if not stripped:
            continue
        if stripped == "mappings:":
            in_mappings = True
            mappings_indent = indent
            continue
        if (
            in_mappings
            and stripped.endswith(":")
            and not stripped.startswith("-")
            and not stripped.startswith("s:")
            and mappings_indent is not None
            and indent == mappings_indent + 2
        ):
            name = stripped[:-1].strip()
            if name in ignore_keys:
                continue
            current = {"name": name, "type": None, "props": []}
            mappings.append(current)
            last_p = None
            pending_o = None
            in_props_block = False
            continue
        if current is None:
            continue
        if stripped == "sources:":
            in_sources = True
            sources_indent = indent
            continue
        if in_sources:
            if sources_indent is not None and indent > sources_indent:
                continue
            in_sources = False
            sources_indent = None
        if stripped == "po:":
            continue
        if stripped == "p:":
            in_props_block = True
            props_indent = indent
            pending_o = None
            last_p = None
            continue
        if in_props_block and indent <= props_indent and stripped != "p:":
            in_props_block = False
            pending_o = None
        if stripped.startswith("- [") and "value:" in stripped:
            if current is None:
                continue
            match = re.search(r"value:\s*\"([^\"]+)\"", stripped) or re.search(
                r"value:\s*'([^']+)'", stripped
            )
            if match:
                current["source_xpath"] = match.group(1)
            else:
                match = re.search(r"\\['html'\\]\\s*,\\s*\"([^\"]+)\"", stripped)
                if match:
                    current["source_xpath"] = match.group(1)
            continue
        if stripped.startswith("- ["):
            if current is None:
                continue
            match = re.match(r"- \[\s*\"([^\"]+)\"\s*\]$", stripped) or re.match(
                r"- \[\s*'([^']+)'\s*\]$", stripped
            )
            if match:
                current["source_xpath"] = match.group(1)
                continue
        if stripped.startswith("s:") and stripped.endswith(":") and stripped != "s:":
            prop_name = stripped[2:-1].strip()
            if prop_name:
                pending_o = prop_name
            continue
        if stripped.startswith("s:") and not stripped.startswith("s: "):
            rest = stripped[2:]
            prop, _, obj_part = rest.partition(":")
            prop = prop.strip()
            obj = _normalize_xpath_literal(_strip_all_quotes(_strip_wrapped_list(obj_part.strip())))
            if prop:
                current["props"].append((prop, obj))
            continue
        if stripped.startswith("s:"):
            value = _strip_all_quotes(_strip_wrapped_list(stripped.split(":", 1)[1].strip()))
            if value.startswith(("http://", "https://")) or _looks_like_xpath(value):
                continue
            if value.startswith("schema:"):
                current["type"] = normalize_type(value)
                continue
            if re.fullmatch(r"[A-Za-z][A-Za-z0-9]+", value):
                current["type"] = value
            continue
        if stripped.startswith("- p:"):
            _, value = stripped.split("p:", 1)
            last_p = _strip_quotes(value.strip())
            pending_o = None
            continue
        if in_props_block and stripped.startswith("schema:") and stripped.endswith(":"):
            pending_o = stripped[:-1].strip()
            continue
        if stripped.startswith("o:") and last_p:
            _, value = stripped.split("o:", 1)
            obj = _normalize_xpath_literal(_strip_all_quotes(_strip_wrapped_list(value.strip())))
            if obj:
                current["props"].append((last_p, obj))
                last_p = None
                pending_o = None
            else:
                pending_o = last_p
            continue
        if stripped.startswith("- [a,") or stripped.startswith("- [ a,"):
            match = re.search(r"'schema:([^']+)'", stripped) or re.search(r"\"schema:([^\"]+)\"", stripped)
            if match:
                current["type"] = normalize_type(match.group(1))
            continue
        if stripped.startswith("value:") and pending_o:
            _, value = stripped.split("value:", 1)
            obj = _normalize_xpath_literal(_first_list_value(value.strip()))
            current["props"].append((pending_o, obj))
            pending_o = None
            last_p = None
            continue
        if stripped.startswith("mapping:") and pending_o:
            _, value = stripped.split("mapping:", 1)
            obj = _normalize_xpath_literal(_strip_all_quotes(_strip_wrapped_list(value.strip())))
            if obj:
                current["props"].append((pending_o, obj))
            pending_o = None
            last_p = None
            continue
        if stripped.startswith("- ["):
            if "p:" in stripped and "o:" in stripped:
                match = re.search(r"\[p:\s*([^,]+),\s*o:\s*(.+)\]$", stripped)
                if match:
                    prop = _strip_quotes(match.group(1).strip())
                    obj = _normalize_xpath_literal(
                        _strip_all_quotes(_strip_wrapped_list(match.group(2).strip()))
                    )
                    if prop == "a":
                        continue
                    current["props"].append((prop, obj))
                continue
            match = re.search(r"\[\s*([^,]+)\s*,\s*(.+)\]$", stripped)
            if match:
                prop = _strip_quotes(match.group(1).strip())
                obj = _normalize_xpath_literal(_strip_all_quotes(_strip_wrapped_list(match.group(2).strip())))
                if prop == "a":
                    continue
                current["props"].append((prop, obj))

    if not mappings:
        raise UsageError("Agent response did not include recognizable mappings.")

    mapping_names = {m["name"] for m in mappings}
    target = normalize_type(target_type) if target_type else None
    denied_props = {"schema:html", "html"}
    schema_props = _schema_property_set()

    main_mapping = None
    for mapping in mappings:
        if target and normalize_type(mapping["type"] or "") == target:
            main_mapping = mapping
            break
    if main_mapping is None:
        main_mapping = mappings[0]
    if main_mapping:
        main_mapping["__main__"] = True

    output_lines = [
        "prefixes:",
        f"  schema: '{_SCHEMA_BASE}/'",
        "  ex: 'http://example.com/'",
        "mappings:",
    ]

    for mapping in mappings:
        map_name = mapping["name"]
        map_type = mapping["type"] or ("Review" if mapping is main_mapping else "Thing")
        map_type = normalize_type(map_type)
        output_lines += [
            f"  {map_name}:",
            "    sources:",
            f"      - [{file_uri}~xpath, '/']",
            f"    s: ex:{map_name}~iri",
            "    po:",
            f"      - [a, 'schema:{map_type}']",
        ]
        props = list(mapping["props"])
        source_xpath = mapping.get("source_xpath")

        if mapping is main_mapping:
            has_url = any(p == "schema:url" for p, _ in props)
            if not has_url:
                props.insert(0, ("schema:url", url))

        for prop, obj in props:
            if not prop.startswith("schema:"):
                prop = f"schema:{prop}"
            prop_name = prop[7:]
            if prop_name == "a" or prop == "schema:a":
                continue
            if "~" in prop_name or "http" in prop_name:
                continue
            if prop in denied_props:
                continue
            if prop_name not in schema_props:
                continue
            if not obj:
                continue
            if obj == "{value}" and source_xpath:
                obj = source_xpath
            if isinstance(obj, str) and obj.startswith("ex:") and obj[3:] in mapping_names:
                obj = obj[3:]
            if obj in mapping_names:
                output_lines.append(f"      - [{prop}, ex:{obj}~iri]")
                continue
            if _looks_like_xpath(obj):
                xpath = _normalize_xpath_reference(_simplify_xpath(obj)).replace("'", '"')
                output_lines.append(f"      - [{prop}, '$(%s)']" % xpath)
                continue
            output_lines.append(f"      - [{prop}, '{obj}']")

    return "\n".join(output_lines) + "\n", mappings


def _quote_unquoted_xpath_attributes(text: str) -> str:
    pattern = re.compile(
        r"@(id|class|property|rel|name|type|itemprop|content|href|src)\s*=\s*([A-Za-z0-9_-]+)"
    )

    def repl(match: re.Match[str]) -> str:
        attr = match.group(1)
        value = match.group(2)
        return f'@{attr}="{value}"'

    return pattern.sub(repl, text)


def _run_yarrrml_parser(input_path: Path, output_path: Path) -> None:
    parser = shutil.which("yarrrml-parser")
    if not parser:
        raise UsageError(
            "yarrrml-parser is required. Install with: npm install -g @rmlio/yarrrml-parser"
        )
    if output_path.exists():
        output_path.unlink()
    result = subprocess.run(
        [parser, "-i", str(input_path), "-o", str(output_path)],
        capture_output=True,
        text=True,
        check=False,
    )
    if not output_path.exists():
        error = (result.stderr or result.stdout).strip()
        raise UsageError(f"yarrrml-parser failed to produce output. {error}")
    if result.returncode != 0:
        raise UsageError(f"yarrrml-parser failed: {result.stderr.strip()}")


def _materialize_graph(mapping_path: Path) -> Graph:
    try:
        import morph_kgc
    except ImportError as exc:
        raise UsageError("morph-kgc is required. Install with: pip install morph-kgc") from exc

    config = (
        "[CONFIGURATION]\n"
        "output_format = N-TRIPLES\n"
        "\n"
        "[DataSource1]\n"
        f"mappings = {mapping_path}\n"
    )
    return morph_kgc.materialize(config)


def materialize_yarrrml(
    yarrrml: str,
    xhtml_path: Path,
    workdir: Path,
    *,
    url: str | None = None,
) -> Graph:
    file_uri = xhtml_path.as_posix()
    normalized = _replace_sources_with_file(yarrrml, file_uri)
    if url:
        normalized = re.sub(
            r"(schema:url\s*,\s*['\"])__URL__(['\"])",
            rf"\1{url}\2",
            normalized,
        )
    workdir.mkdir(parents=True, exist_ok=True)
    yarrml_path = workdir / "mapping.yarrrml"
    rml_path = workdir / "mapping.ttl"
    yarrml_path.write_text(normalized)
    _run_yarrrml_parser(yarrml_path, rml_path)
    _ensure_subject_termtype_iri(rml_path)
    _normalize_reference_formulation(rml_path)
    return _materialize_graph(rml_path)


def normalize_yarrrml_mappings(
    yarrrml: str,
    url: str,
    xhtml_path: Path,
    target_type: str | None = None,
) -> tuple[str, list[dict[str, Any]]]:
    return _normalize_agent_yarrml(yarrrml, url, xhtml_path.as_posix(), target_type)


def materialize_yarrrml_jsonld(
    yarrrml: str,
    xhtml_path: Path,
    workdir: Path,
    *,
    url: str | None = None,
) -> dict[str, Any] | list[Any]:
    file_uri = xhtml_path.as_posix()
    normalized = _replace_sources_with_file(yarrrml, file_uri)
    if url:
        normalized = re.sub(
            r"(schema:url\s*,\s*['\"])__URL__(['\"])",
            rf"\1{url}\2",
            normalized,
        )
    workdir.mkdir(parents=True, exist_ok=True)
    yarrml_path = workdir / "mapping.yarrml"
    rml_path = workdir / "mapping.ttl"
    yarrml_path.write_text(normalized)
    _run_yarrrml_parser(yarrml_path, rml_path)
    _ensure_subject_termtype_iri(rml_path)
    _normalize_reference_formulation(rml_path)
    return _materialize_jsonld(rml_path)


def postprocess_jsonld(
    jsonld_raw: dict[str, Any] | list[Any],
    mappings: list[dict[str, Any]],
    xhtml: str,
    dataset_uri: str,
    url: str,
    target_type: str | None = None,
) -> dict[str, Any]:
    jsonld_raw = _fill_jsonld_from_mappings(jsonld_raw, mappings, xhtml)
    _ensure_node_ids(jsonld_raw, dataset_uri, url)
    _dedupe_review_notes(jsonld_raw)
    normalized = normalize_jsonld(jsonld_raw, dataset_uri, url, target_type, embed_nodes=False)
    _materialize_literal_nodes(normalized, dataset_uri, url)
    _ensure_author_name(normalized, xhtml, dataset_uri, url)
    _ensure_review_url(normalized, url)
    _prune_empty_rating_nodes(normalized)
    return normalized


def _prune_empty_rating_nodes(data: dict[str, Any] | list[Any]) -> None:
    nodes = _flatten_jsonld(data)
    if not nodes:
        return
    rating_value_key = f"{_SCHEMA_BASE}/ratingValue"
    empty_rating_ids: set[str] = set()

    def _has_rating_value(node: dict[str, Any]) -> bool:
        value = node.get(rating_value_key, node.get("ratingValue"))
        if value is None:
            return False
        values = value if isinstance(value, list) else [value]
        for item in values:
            if isinstance(item, dict):
                text = item.get("@value") or item.get("value")
            else:
                text = item
            if isinstance(text, str) and text.strip():
                return True
            if isinstance(text, (int, float)):
                return True
        return False

    for node in nodes:
        if not isinstance(node, dict):
            continue
        node_types = {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
        if "Rating" not in node_types:
            continue
        if not _has_rating_value(node):
            node_id = node.get("@id")
            if isinstance(node_id, str):
                empty_rating_ids.add(node_id)
    if not empty_rating_ids:
        return

    def _filter_refs(values: Any) -> Any:
        if isinstance(values, list):
            filtered = [
                value
                for value in values
                if not (isinstance(value, dict) and value.get("@id") in empty_rating_ids)
            ]
            return filtered
        return values

    for node in nodes:
        if not isinstance(node, dict):
            continue
        for key in (f"{_SCHEMA_BASE}/reviewRating", "reviewRating"):
            if key in node:
                node[key] = _filter_refs(node[key])
                if not node[key]:
                    node.pop(key, None)

    if isinstance(data, dict) and isinstance(data.get("@graph"), list):
        data["@graph"] = [
            node
            for node in data["@graph"]
            if not (isinstance(node, dict) and node.get("@id") in empty_rating_ids)
        ]
    elif isinstance(data, list):
        data[:] = [
            node
            for node in data
            if not (isinstance(node, dict) and node.get("@id") in empty_rating_ids)
        ]


def _dedupe_review_notes(data: dict[str, Any] | list[Any]) -> None:
    nodes = _flatten_jsonld(data)
    if not nodes:
        return
    pos_key = f"{_SCHEMA_BASE}/positiveNotes"
    neg_key = f"{_SCHEMA_BASE}/negativeNotes"

    def _extract_values(values: Any) -> list[str]:
        if isinstance(values, list):
            items = values
        else:
            items = [values]
        normalized: list[str] = []
        for item in items:
            if isinstance(item, dict):
                value = item.get("@value") or item.get("value")
            else:
                value = item
            if isinstance(value, str):
                normalized.append(value.strip())
        return [value for value in normalized if value]

    for node in nodes:
        if not isinstance(node, dict):
            continue
        node_types = {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
        if "Review" not in node_types and "Product" not in node_types:
            continue
        pos_values = _extract_values(node.get(pos_key) or node.get("positiveNotes"))
        neg_values = _extract_values(node.get(neg_key) or node.get("negativeNotes"))
        if pos_values and neg_values and pos_values == neg_values:
            node.pop(pos_key, None)
            node.pop("positiveNotes", None)


def _materialize_literal_nodes(
    data: dict[str, Any] | list[Any],
    dataset_uri: str,
    url: str,
) -> None:
    nodes = _flatten_jsonld(data)
    if not nodes:
        return
    graph = data.get("@graph") if isinstance(data, dict) else None
    if not isinstance(graph, list):
        return
    schema_author = f"{_SCHEMA_BASE}/author"
    schema_item = f"{_SCHEMA_BASE}/itemReviewed"
    schema_publisher = f"{_SCHEMA_BASE}/publisher"
    schema_name = f"{_SCHEMA_BASE}/name"

    def _ensure_node(type_name: str, name: str, index: int) -> dict[str, Any]:
        node_id = build_id_base(dataset_uri, type_name, name, url, index)
        node = {
            "@id": node_id,
            "@type": [f"{_SCHEMA_BASE}/{type_name}"],
            schema_name: [{"@value": name}],
            "@context": _SCHEMA_BASE,
        }
        graph.append(node)
        return node

    def _replace_literal(node: dict[str, Any], key: str, type_name: str, start_index: int) -> None:
        values = node.get(key)
        if not values:
            return
        items = values if isinstance(values, list) else [values]
        new_refs: list[dict[str, Any]] = []
        for idx, item in enumerate(items, start=start_index):
            if isinstance(item, dict) and item.get("@id"):
                new_refs.append(item)
                continue
            if isinstance(item, dict) and "@value" in item:
                name = str(item["@value"]).strip()
            else:
                name = str(item).strip()
            if not name:
                continue
            new_node = _ensure_node(type_name, name, idx)
            new_refs.append({"@id": new_node["@id"]})
        if new_refs:
            node[key] = new_refs
        else:
            node.pop(key, None)

    review_nodes = [
        node
        for node in nodes
        if isinstance(node, dict)
        and "Review" in {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
    ]
    for index, review in enumerate(review_nodes, start=1):
        _replace_literal(review, schema_author, "Person", index)
        _replace_literal(review, schema_item, "Product", index + 100)
        _replace_literal(review, schema_publisher, "Organization", index + 200)


def _ensure_review_url(data: dict[str, Any] | list[Any], url: str) -> None:
    nodes = _flatten_jsonld(data)
    if not nodes or not url:
        return
    url_key = f"{_SCHEMA_BASE}/url"
    for node in nodes:
        if not isinstance(node, dict):
            continue
        node_types = {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
        if "Review" not in node_types:
            continue
        if url_key not in node:
            node[url_key] = [{"@value": url}]


def _ensure_author_name(
    data: dict[str, Any] | list[Any],
    xhtml: str,
    dataset_uri: str,
    url: str,
) -> None:
    author_name = _extract_author_name(xhtml)
    if not author_name:
        return
    nodes = _flatten_jsonld(data)
    if not nodes:
        return
    schema_name = f"{_SCHEMA_BASE}/name"
    schema_author = f"{_SCHEMA_BASE}/author"
    graph = data.get("@graph") if isinstance(data, dict) else None
    if not isinstance(graph, list):
        return

    author_nodes = [
        node
        for node in nodes
        if isinstance(node, dict)
        and "Person" in {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
    ]
    for node in author_nodes:
        if schema_name not in node:
            node[schema_name] = [{"@value": author_name}]

    review_nodes = [
        node
        for node in nodes
        if isinstance(node, dict)
        and "Review" in {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
    ]
    for review in review_nodes:
        if schema_author in review:
            continue
        author_node = build_id_base(dataset_uri, "Person", author_name, url, 0)
        graph.append(
            {
                "@id": author_node,
                "@type": [f"{_SCHEMA_BASE}/Person"],
                schema_name: [{"@value": author_name}],
                "@context": _SCHEMA_BASE,
            }
        )
        review[schema_author] = [{"@id": author_node}]


def _extract_author_name(xhtml: str) -> str | None:
    try:
        from lxml import html as lxml_html
    except Exception:
        return None
    parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
    try:
        doc = lxml_html.document_fromstring(xhtml, parser=parser)
    except Exception:
        return None

    candidates = [
        "//meta[@name='author']/@content",
        "//meta[@property='article:author']/@content",
        "//a[@rel='author']/text()",
        "//*[contains(normalize-space(.), 'Written by')]/following::a[1]/text()",
    ]
    for path in candidates:
        try:
            results = doc.xpath(path)
        except Exception:
            continue
        for value in results:
            if isinstance(value, str) and value.strip():
                return value.strip()
    return None

def _materialize_jsonld(mapping_path: Path) -> dict[str, Any] | list[Any]:
    graph = _materialize_graph(mapping_path)
    jsonld_str = graph.serialize(format="json-ld")
    return json.loads(jsonld_str)


def _ensure_subject_termtype_iri(mapping_path: Path) -> None:
    graph = Graph()
    graph.parse(mapping_path, format="turtle")
    for subject_map in graph.subjects(RDF.type, _RR.SubjectMap):
        graph.add((subject_map, _RR.termType, _RR.IRI))
    graph.serialize(destination=str(mapping_path), format="turtle")


def _normalize_reference_formulation(mapping_path: Path) -> None:
    graph = Graph()
    graph.parse(mapping_path, format="turtle")
    replaced = False
    for predicate in (_RML.referenceFormulation, _RML_LEGACY.referenceFormulation):
        for subject in list(graph.subjects(predicate, _QL.XPath)):
            graph.remove((subject, predicate, _QL.XPath))
            graph.add((subject, predicate, _RML.XPath))
            replaced = True
    if replaced:
        graph.serialize(destination=str(mapping_path), format="turtle")


def _flatten_jsonld(data: dict[str, Any] | list[Any]) -> list[dict[str, Any]]:
    if isinstance(data, list):
        return [node for node in data if isinstance(node, dict)]
    if "@graph" in data and isinstance(data["@graph"], list):
        return [node for node in data["@graph"] if isinstance(node, dict)]
    return [data] if isinstance(data, dict) else []


def ensure_no_blank_nodes(graph: Graph) -> None:
    offenders: list[tuple[Identifier, Identifier, Identifier]] = []
    for subj, pred, obj in graph:
        if isinstance(subj, BNode) or isinstance(obj, BNode):
            offenders.append((subj, pred, obj))
            if len(offenders) >= 5:
                break
    if offenders:
        sample = "; ".join(f"{s} {p} {o}" for s, p, o in offenders)
        raise UsageError(
            "Blank nodes are not allowed in RDF output. "
            f"Found {len(offenders)} sample triples with blank nodes: {sample}"
        )


def _collect_jsonld_nodes(data: Any) -> list[dict[str, Any]]:
    nodes: list[dict[str, Any]] = []

    def _walk(value: Any) -> None:
        if isinstance(value, dict):
            if _is_jsonld_node(value):
                nodes.append(value)
            for child in value.values():
                _walk(child)
        elif isinstance(value, list):
            for item in value:
                _walk(item)

    _walk(data)
    return nodes


def _strip_iri_suffix(value: str) -> str:
    return value[:-4] if value.endswith("~iri") else value


def _normalize_iri_suffixes(data: Any) -> Any:
    if isinstance(data, dict):
        out: dict[str, Any] = {}
        for key, value in data.items():
            if key == "@id" and isinstance(value, str):
                out[key] = _strip_iri_suffix(value)
            else:
                out[key] = _normalize_iri_suffixes(value)
        return out
    if isinstance(data, list):
        return [_normalize_iri_suffixes(item) for item in data]
    return data


def _xpath_first_text(doc: Any, xpath: str) -> str | None:
    def _eval(path: str) -> list[Any]:
        return doc.xpath(path)

    try:
        result = _eval(xpath)
    except Exception:
        return None
    if not result:
        relaxed = _relax_xpath(xpath)
        if relaxed != xpath:
            try:
                result = _eval(relaxed)
            except Exception:
                return None
        if not result:
            return None
    for item in result:
        if isinstance(item, str):
            text = item.strip()
        elif hasattr(item, "text_content"):
            text = item.text_content().strip()
        else:
            text = str(item).strip()
        if text:
            return text
    return None


def _relax_xpath(value: str) -> str:
    relaxed = value
    relaxed = re.sub(r'@class=\"([^\"]+)\"', r'contains(@class, "\1")', relaxed)
    relaxed = re.sub(r'@id=\"([^\"]+)\"', r'contains(@id, "\1")', relaxed)
    relaxed = relaxed.replace("//div[", "//*[")
    relaxed = relaxed.replace("/div[", "/*[")
    relaxed = relaxed.replace("//p[", "//*[")
    relaxed = relaxed.replace("/p[", "/*[")
    return relaxed


def _extract_list_items(doc: Any, xpaths: list[str]) -> list[str]:
    items: list[str] = []
    seen: set[str] = set()
    for path in xpaths:
        try:
            results = doc.xpath(path)
        except Exception:
            continue
        for item in results:
            if hasattr(item, "text_content"):
                text = item.text_content().strip()
            else:
                text = str(item).strip()
            if text:
                if text in seen:
                    continue
                seen.add(text)
                items.append(text)
    return items


def _build_item_list(items: list[str]) -> dict[str, Any]:
    entries = []
    for idx, name in enumerate(items, start=1):
        entries.append(
            {
                "@type": "ListItem",
                "position": idx,
                "name": name,
            }
        )
    return {"@type": "ItemList", "itemListElement": entries}


def _is_item_list_value(value: Any) -> bool:
    if not isinstance(value, list) or not value:
        return False
    item = value[0]
    if not isinstance(item, dict):
        return False
    return normalize_type(item.get("@type")) == "ItemList"


def _extract_rating_number(text: str | None) -> str | None:
    if not text:
        return None
    match = re.search(r"-?\\d+(?:\\.\\d+)?", text)
    if not match:
        return None
    try:
        value = float(match.group(0))
    except ValueError:
        return None
    if value < 0 or value > 5:
        return None
    return match.group(0)


def _extract_rating_value(doc: Any) -> str | None:
    candidates = [
        "//*[@itemprop='ratingValue']/text()",
        "//*[@data-rating]/@data-rating",
        "//*[contains(@class, 'rating')]/text()",
        "//*[contains(@class, 'Rating')]/text()",
        "//*[contains(@id, 'rating')]/text()",
        "//*[contains(@id, 'Rating')]/text()",
        "//*[contains(@aria-label, 'rating')]/@aria-label",
        "//*[contains(@aria-label, 'star')]/@aria-label",
    ]
    for xpath in candidates:
        text = _xpath_first_text(doc, xpath)
        value = _extract_rating_number(text)
        if value is not None:
            return value
    return None


def enrich_graph_from_xhtml(graph: Graph, xhtml: str, url: str | None = None) -> None:
    try:
        from lxml import html as lxml_html
    except Exception:
        return
    parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
    try:
        doc = lxml_html.document_fromstring(xhtml, parser=parser)
    except Exception:
        return

    schema = Namespace(f"{_SCHEMA_BASE}/")
    review_type = URIRef(f"{_SCHEMA_BASE}/Review")
    review_nodes = list(graph.subjects(RDF.type, review_type))
    if not review_nodes:
        return

    title = (
        _xpath_first_text(doc, "/html/head/meta[@property=\"og:title\"]/@content")
        or _xpath_first_text(doc, "/html/head/title/text()")
        or _xpath_first_text(doc, "//h1[1]")
    )
    description = _xpath_first_text(doc, "/html/head/meta[@property=\"og:description\"]/@content") or _xpath_first_text(
        doc, "/html/head/meta[@name=\"description\"]/@content"
    )
    author_name = (
        _xpath_first_text(doc, "/html/head/meta[@name=\"author\"]/@content")
        or _xpath_first_text(doc, "/html/head/meta[@property=\"author\"]/@content")
        or _xpath_first_text(doc, "/html/head/meta[@property=\"article:author\"]/@content")
    )
    item_name = _xpath_first_text(doc, "//figure//img/@alt") or _xpath_first_text(doc, "//h1[1]")

    for review in review_nodes:
        if url and graph.value(review, schema.url) is None:
            graph.add((review, schema.url, Literal(url)))
        if title and graph.value(review, schema.name) is None:
            graph.add((review, schema.name, Literal(title)))
        if description and graph.value(review, schema.description) is None:
            graph.add((review, schema.description, Literal(description)))

        author = graph.value(review, schema.author)
        if author is not None and author_name and graph.value(author, schema.name) is None:
            graph.add((author, schema.name, Literal(author_name)))

        item = graph.value(review, schema.itemReviewed)
        if item is not None and item_name and graph.value(item, schema.name) is None:
            graph.add((item, schema.name, Literal(item_name)))

        rating = graph.value(review, schema.reviewRating)
        if rating is not None and graph.value(rating, schema.ratingValue) is None:
            rating_value = _extract_rating_value(doc)
            if rating_value:
                graph.add((rating, schema.ratingValue, Literal(rating_value)))


def _fill_jsonld_from_mappings(
    data: dict[str, Any] | list[Any],
    mappings: list[dict[str, Any]],
    xhtml: str,
) -> dict[str, Any] | list[Any]:
    try:
        from lxml import html as lxml_html
    except Exception:
        return data
    parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
    try:
        doc = lxml_html.document_fromstring(xhtml, parser=parser)
    except Exception:
        return data

    nodes = _flatten_jsonld(data)
    node_by_id: dict[str, dict[str, Any]] = {
        str(node.get("@id")): node for node in nodes if isinstance(node, dict) and node.get("@id")
    }
    for node_id, node in list(node_by_id.items()):
        if node_id.endswith("~iri"):
            node_by_id.setdefault(node_id[: -len("~iri")], node)

    def _author_url_fallback() -> str | None:
        return _xpath_first_text(doc, "/html/head/link[@rel='author']/@href")

    author_url_fallback = _author_url_fallback()

    for mapping in mappings:
        name = mapping.get("name")
        if not name:
            continue
        node_id = f"http://example.com/{name}~iri"
        node = node_by_id.get(node_id) or node_by_id.get(f"http://example.com/{name}")
        if node is None:
            continue
        for prop, obj in mapping.get("props", []):
            prop_name = prop[7:] if prop.startswith("schema:") else prop
            if prop_name in {"a", "url"}:
                continue
            full_prop = f"{_SCHEMA_BASE}/{prop_name}"
            if full_prop in node:
                continue
            if not obj:
                continue
            if obj.startswith("ex:") and obj.endswith("~iri"):
                target = obj.split("ex:", 1)[1].split("~", 1)[0]
                node[full_prop] = [{"@id": f"http://example.com/{target}"}]
                continue
            if _looks_like_xpath(obj):
                xpath = _normalize_xpath_reference(_simplify_xpath(obj))
                text = _xpath_first_text(doc, xpath)
                if text:
                    if prop_name in {"ratingValue", "bestRating", "worstRating"}:
                        match = re.search(r"-?\\d+(?:\\.\\d+)?", text)
                        if not match:
                            continue
                        text = match.group(0)
                    node[full_prop] = [{"@value": text}]
                    if prop_name == "name":
                        node_type = node.get("@type") or []
                        node_types = {normalize_type(t) for t in node_type if isinstance(t, str)}
                        if node_types & {"Person", "Organization"}:
                            url_xpath = f"{xpath}/@href"
                            url_value = _xpath_first_text(doc, url_xpath)
                            if url_value:
                                node[f"{_SCHEMA_BASE}/url"] = [{"@value": url_value}]
                            elif author_url_fallback:
                                node[f"{_SCHEMA_BASE}/url"] = [{"@value": author_url_fallback}]
                continue
            if obj:
                node[full_prop] = [{"@value": obj}]

    for node in nodes:
        if not isinstance(node, dict):
            continue
        node_types = {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
        if "Review" not in node_types:
            continue
        review_rating_prop = f"{_SCHEMA_BASE}/reviewRating"
        review_rating_pairs: list[tuple[dict[str, Any], dict[str, Any]]] = []
        for rating_ref in node.get(review_rating_prop, []):
            if isinstance(rating_ref, dict) and rating_ref.get("@id"):
                rating_node = node_by_id.get(str(rating_ref["@id"]))
                if isinstance(rating_node, dict):
                    review_rating_pairs.append((rating_ref, rating_node))
        valid_rating_refs: list[dict[str, Any]] = []
        for rating_ref, rating_node in review_rating_pairs:
            rating_value_key = f"{_SCHEMA_BASE}/ratingValue"
            if rating_value_key not in rating_node:
                rating_value = _extract_rating_value(doc)
                if rating_value:
                    rating_node[rating_value_key] = [{"@value": rating_value}]
            if rating_value_key in rating_node:
                valid_rating_refs.append(rating_ref)
        if review_rating_pairs:
            if valid_rating_refs:
                node[review_rating_prop] = valid_rating_refs
            else:
                node.pop(review_rating_prop, None)
        if f"{_SCHEMA_BASE}/description" not in node:
            description = _xpath_first_text(
                doc, "/html/head/meta[@property=\"og:description\"]/@content"
            ) or _xpath_first_text(doc, "/html/head/meta[@name=\"description\"]/@content")
            if description:
                node[f"{_SCHEMA_BASE}/description"] = [{"@value": description}]
    return data


def _extract_type(node: dict[str, Any]) -> str | None:
    raw = node.get("@type")
    if isinstance(raw, list) and raw:
        raw = raw[0]
    if isinstance(raw, str):
        return normalize_type(raw)
    return None


def _extract_name(node: dict[str, Any]) -> str | None:
    for key in ("name", "headline", "title"):
        value = node.get(key)
        if isinstance(value, str) and value.strip():
            return value.strip()
    return None


def _extract_text_value(value: Any) -> str | None:
    if isinstance(value, str) and value.strip():
        return value.strip()
    if isinstance(value, dict):
        raw = value.get("@value") or value.get("@id")
        if isinstance(raw, str) and raw.strip():
            return raw.strip()
    if isinstance(value, list):
        for item in value:
            text = _extract_text_value(item)
            if text:
                return text
    return None


def _extract_name_any(node: dict[str, Any]) -> str | None:
    name = _extract_name(node)
    if name:
        return name
    for key in (f"{_SCHEMA_BASE}/name", f"{_SCHEMA_BASE}/headline", f"{_SCHEMA_BASE}/title"):
        value = _extract_text_value(node.get(key))
        if value:
            return value
    return None


def _extract_url_any(node: dict[str, Any]) -> str | None:
    value = node.get("url")
    text = _extract_text_value(value)
    if text:
        return text
    value = node.get(f"{_SCHEMA_BASE}/url")
    text = _extract_text_value(value)
    if text:
        return text
    return None


def _local_prop_name(name: str) -> str:
    if name.startswith(_SCHEMA_BASE):
        return name.rsplit("/", 1)[-1]
    if name.startswith("schema:"):
        return name.split(":", 1)[-1]
    return name


_INDEPENDENT_PROPERTIES = {
    "author",
    "creator",
    "publisher",
    "editor",
    "contributor",
    "copyrightHolder",
    "brand",
    "manufacturer",
    "provider",
    "seller",
    "organizer",
    "performer",
    "actor",
    "director",
    "producer",
    "member",
    "memberOf",
    "affiliation",
    "parentOrganization",
    "subOrganization",
    "alumniOf",
    "sponsor",
    "about",
    "mentions",
    "mainEntity",
    "mainEntityOfPage",
    "isPartOf",
    "partOfSeries",
    "location",
    "areaServed",
}


def _is_jsonld_node(node: dict[str, Any]) -> bool:
    if "@type" in node:
        return True
    return any(isinstance(key, str) and key.startswith(_SCHEMA_BASE) for key in node)


def _ensure_node_ids(
    data: dict[str, Any] | list[Any],
    dataset_uri: str,
    url: str,
) -> None:
    seen: set[str] = set()
    replacements: dict[str, str] = {}

    def _collect(value: Any) -> None:
        if isinstance(value, dict):
            node_id = value.get("@id")
            if isinstance(node_id, str) and node_id and not node_id.startswith("_:"):
                seen.add(node_id)
            for child in value.values():
                _collect(child)
        elif isinstance(value, list):
            for item in value:
                _collect(item)

    def _assign(
        value: Any,
        counter: list[int],
        parent_id: str | None = None,
        prop_name: str | None = None,
    ) -> None:
        if isinstance(value, dict):
            if _is_jsonld_node(value):
                node_id = value.get("@id")
                local_prop = _local_prop_name(prop_name or "")
                use_parent = bool(parent_id and local_prop and local_prop not in _INDEPENDENT_PROPERTIES)
                base_uri = parent_id if use_parent else dataset_uri
                needs_id = (
                    not isinstance(node_id, str)
                    or not node_id
                    or node_id.startswith("_:")
                    or (use_parent and not node_id.startswith(parent_id or ""))
                    or (not use_parent and not node_id.startswith(dataset_uri))
                )
                if needs_id:
                    type_name = _extract_type(value) or "Thing"
                    name = _extract_name_any(value)
                    if type_name == "ListItem":
                        position = _extract_text_value(value.get("position")) or _extract_text_value(
                            value.get(f"{_SCHEMA_BASE}/position")
                        )
                        if position:
                            name = f"item-{position}"
                    if not name:
                        name = _dash_type(type_name)
                    node_url = _extract_url_any(value)
                    base_id = build_id_base(base_uri, type_name, name, node_url, counter[0])
                    candidate = base_id
                    suffix = 1
                    while candidate in seen:
                        suffix += 1
                        candidate = f"{base_id}-{suffix}"
                    if isinstance(node_id, str) and node_id and node_id != candidate:
                        replacements[node_id] = candidate
                        if node_id.endswith("~iri"):
                            replacements[node_id[: -len("~iri")]] = candidate
                        else:
                            replacements[f"{node_id}~iri"] = candidate
                    value["@id"] = candidate
                    seen.add(candidate)
                    counter[0] += 1
            current_id = value.get("@id") if isinstance(value.get("@id"), str) else parent_id
            for key, child in value.items():
                if key in ("@id", "@type"):
                    continue
                _assign(child, counter, current_id if isinstance(current_id, str) else None, key)
        elif isinstance(value, list):
            for item in value:
                _assign(item, counter, parent_id, prop_name)

    def _replace(value: Any) -> None:
        if isinstance(value, dict):
            node_id = value.get("@id")
            if isinstance(node_id, str) and node_id in replacements:
                value["@id"] = replacements[node_id]
            for child in value.values():
                _replace(child)
        elif isinstance(value, list):
            for item in value:
                _replace(item)

    _collect(data)
    _assign(data, [1])
    if replacements:
        _replace(data)


def _blank_node_errors(data: dict[str, Any] | list[Any]) -> list[str]:
    errors: list[str] = []

    def _walk(value: Any) -> None:
        if isinstance(value, dict):
            if _is_jsonld_node(value):
                node_id = value.get("@id")
                if not isinstance(node_id, str) or not node_id or node_id.startswith("_:"):
                    errors.append("JSON-LD node missing @id or uses a blank node identifier")
            for child in value.values():
                _walk(child)
        elif isinstance(value, list):
            for item in value:
                _walk(item)

    _walk(data)
    return errors


def _review_rating_dropped(
    data: dict[str, Any] | list[Any],
    mappings: list[dict[str, Any]],
    target_type: str | None,
) -> bool:
    target = normalize_type(target_type or "Thing")
    if target != "Review":
        return False
    mapped_props = _main_mapping_props(mappings)
    if "reviewRating" not in mapped_props:
        return False
    nodes = _flatten_jsonld(data)
    for node in nodes:
        if not isinstance(node, dict):
            continue
        node_types = {normalize_type(t) for t in node.get("@type", []) if isinstance(t, str)}
        if "Review" in node_types:
            return f"{_SCHEMA_BASE}/reviewRating" not in node
    return False


def _build_id_map(
    nodes: list[dict[str, Any]],
    dataset_uri: str,
    url: str,
) -> dict[str, str]:
    id_map: dict[str, str] = {}
    for idx, node in enumerate(nodes):
        old_id = node.get("@id") or f"_:b{idx}"
        type_name = _extract_type(node) or "Thing"
        name = _extract_name_any(node) or _dash_type(type_name)
        node_url = _extract_url_any(node)
        if isinstance(old_id, str) and old_id.startswith(dataset_uri):
            new_id = old_id
        else:
            new_id = build_id(dataset_uri, type_name, name, node_url, idx + 1)
        if old_id in id_map:
            new_id = f"{new_id}-{idx}"
        id_map[str(old_id)] = new_id
    return id_map


def _rewrite_refs(
    value: Any,
    id_map: dict[str, str],
    node_map: dict[str, dict[str, Any]],
    *,
    embed_nodes: bool,
) -> Any:
    if isinstance(value, dict):
        if "@id" in value and isinstance(value["@id"], str):
            ref_id = id_map.get(value["@id"], value["@id"])
            if embed_nodes and ref_id in node_map:
                return node_map[ref_id]
            return {"@id": ref_id}
        return {k: _rewrite_refs(v, id_map, node_map, embed_nodes=embed_nodes) for k, v in value.items()}
    if isinstance(value, list):
        return [_rewrite_refs(item, id_map, node_map, embed_nodes=embed_nodes) for item in value]
    return value


def _main_mapping_props(mappings: list[dict[str, Any]]) -> set[str]:
    schema_props = _schema_property_set()
    for mapping in mappings:
        if mapping.get("__main__"):
            props = mapping.get("props") or []
            clean: set[str] = set()
            for prop, _ in props:
                if not isinstance(prop, str):
                    continue
                name = prop[7:] if prop.startswith("schema:") else prop
                if "~" in name or "http" in name or name == "a":
                    continue
                base = name.split(".", 1)[0]
                if base in schema_props or name in schema_props:
                    clean.add(base)
            return clean
    return set()


def _missing_required_props(
    required_props: list[str],
    mapped_props: set[str],
) -> list[str]:
    missing: set[str] = set()
    mapped_base = {prop.split(".", 1)[0] for prop in mapped_props}
    for prop in required_props:
        base = prop.split(".", 1)[0]
        if base not in mapped_base:
            missing.add(prop)
    return sorted(missing)


def _missing_recommended_props(
    recommended_props: list[str],
    mapped_props: set[str],
) -> list[str]:
    missing: set[str] = set()
    mapped_base = {prop.split(".", 1)[0] for prop in mapped_props}
    for prop in recommended_props:
        base = prop.split(".", 1)[0]
        if base not in mapped_base:
            missing.add(prop)
    return sorted(missing)


def _google_allowed_properties(property_guides: dict[str, dict[str, list[str]]]) -> dict[str, list[str]]:
    allowed: dict[str, list[str]] = {}
    for type_name, guide in property_guides.items():
        props = set(guide.get("required", [])) | set(guide.get("recommended", []))
        if type_name == "Review":
            props |= _REVIEW_OPTIONAL_EXTRAS
        props = sorted(props)
        allowed[type_name] = props
    return allowed


def _mapping_allowed_property_set(
    property_guides: dict[str, dict[str, list[str]]],
) -> set[str]:
    props: set[str] = set()
    for guide in property_guides.values():
        for name in guide.get("required", []) + guide.get("recommended", []):
            props.add(name.split(".", 1)[0])
    props |= _REVIEW_OPTIONAL_EXTRAS
    return props


def _mapping_violations(
    mappings: list[dict[str, Any]],
    allowed_props: set[str],
    target_type: str,
) -> list[str]:
    errors: list[str] = []
    for mapping in mappings:
        map_name = mapping.get("name", "mapping")
        map_type = normalize_type(mapping.get("type") or "")
        for prop, obj in mapping.get("props", []):
            prop_name = prop[7:] if prop.startswith("schema:") else prop
            base = prop_name.split(".", 1)[0]
            if base in {"a", "url"}:
                continue
            if "/" in base or base.startswith("http"):
                continue
            if base not in allowed_props:
                errors.append(f"{map_name}: property not allowed by Google: {base}")
            if _looks_like_xpath(obj):
                continue
            if base in {"author", "reviewRating", "itemReviewed"} and obj.startswith("ex:") and obj.endswith("~iri"):
                continue
            if obj.startswith("ex:") and obj.endswith("~iri"):
                continue
            errors.append(f"{map_name}: hard-coded literal for {base} is not allowed")
        if map_type == "Review":
            review_rating = [obj for prop, obj in mapping.get("props", []) if prop.endswith("reviewRating")]
            for obj in review_rating:
                if not (obj.startswith("ex:") and obj.endswith("~iri")):
                    errors.append(f"{map_name}: reviewRating must map to a Rating node")
    return errors


def _xpath_evidence_errors(
    mappings: list[dict[str, Any]],
    xhtml: str,
) -> list[str]:
    try:
        from lxml import html as lxml_html
    except Exception:
        return []
    parser = lxml_html.HTMLParser(encoding="utf-8", recover=True)
    try:
        doc = lxml_html.document_fromstring(xhtml, parser=parser)
    except Exception:
        return []
    errors: list[str] = []
    for mapping in mappings:
        map_name = mapping.get("name", "mapping")
        for prop, obj in mapping.get("props", []):
            prop_name = prop[7:] if prop.startswith("schema:") else prop
            if prop_name == "url":
                continue
            if not _looks_like_xpath(obj):
                continue
            try:
                result = doc.xpath(_simplify_xpath(obj))
            except Exception:
                errors.append(f"{map_name}: invalid XPath for {prop_name}")
                continue
            if not result:
                errors.append(f"{map_name}: XPath returned no results for {prop_name}")
                continue
            if isinstance(result, list) and all((isinstance(item, str) and not item.strip()) for item in result):
                errors.append(f"{map_name}: XPath returned empty text for {prop_name}")
    return errors


def _xpath_reusability_warnings(mappings: list[dict[str, Any]]) -> list[str]:
    warnings: list[str] = []
    id_with_digits = re.compile(r"@id\\s*=\\s*['\"][^'\"]*\\d[^'\"]*['\"]")
    for mapping in mappings:
        map_name = mapping.get("name", "mapping")
        for prop, obj in mapping.get("props", []):
            prop_name = prop[7:] if prop.startswith("schema:") else prop
            if not _looks_like_xpath(obj):
                continue
            candidate = obj.replace("\\\"", "\"").replace("\\'", "'")
            if id_with_digits.search(candidate):
                warnings.append(
                    f"{map_name}: XPath for {prop_name} uses a numeric @id; prefer a reusable selector."
                )
    return warnings


def _mapping_type_sanity(
    mappings: list[dict[str, Any]],
    expected_types: dict[str, tuple[str, ...]],
) -> list[str]:
    errors: list[str] = []
    mapping_types = {m.get("name"): normalize_type(m.get("type") or "") for m in mappings}
    for mapping in mappings:
        map_name = mapping.get("name")
        for prop, obj in mapping.get("props", []):
            prop_name = prop[7:] if prop.startswith("schema:") else prop
            if obj.startswith("ex:") and obj.endswith("~iri"):
                target = obj.split("ex:", 1)[1].split("~", 1)[0]
                expected = expected_types.get(prop_name)
                if expected:
                    actual = mapping_types.get(target, "")
                    if actual and actual not in expected:
                        errors.append(
                            f"{map_name}: {prop_name} must map to {expected}, got {actual}"
                        )
    return errors


def _format_result_path(value: Identifier | None) -> str:
    if value is None:
        return "unknown"
    if isinstance(value, URIRef):
        return _short_schema_name(value) or str(value)
    return str(value)


def _validation_messages(result: ValidationResult, max_items: int = 20) -> tuple[list[str], list[str]]:
    errors: list[str] = []
    warnings: list[str] = []
    count = 0
    for res in result.report_graph.subjects(RDF.type, _SH.ValidationResult):
        severity = result.report_graph.value(res, _SH.resultSeverity)
        message = result.report_graph.value(res, _SH.resultMessage)
        path = result.report_graph.value(res, _SH.resultPath)
        source_shape = result.report_graph.value(res, _SH.sourceShape)
        source_label = result.shape_source_map.get(source_shape, "unknown")
        line = f"{_format_result_path(path)}: {message or 'validation error'} (shape: {source_label})"
        if severity == _SH.Warning:
            warnings.append(line)
        else:
            errors.append(line)
        count += 1
        if count >= max_items:
            break
    return errors, warnings


def _validation_messages_for_types(
    result: ValidationResult,
    allowed_types: set[str],
    max_items: int = 20,
) -> tuple[list[str], list[str]]:
    errors: list[str] = []
    warnings: list[str] = []
    count = 0
    for res in result.report_graph.subjects(RDF.type, _SH.ValidationResult):
        severity = result.report_graph.value(res, _SH.resultSeverity)
        message = result.report_graph.value(res, _SH.resultMessage)
        path = result.report_graph.value(res, _SH.resultPath)
        source_shape = result.report_graph.value(res, _SH.sourceShape)
        source_label = result.shape_source_map.get(source_shape, "unknown")
        focus = result.report_graph.value(res, _SH.focusNode)
        focus_types: set[str] = set()
        if focus is not None:
            for t in result.data_graph.objects(focus, RDF.type):
                if isinstance(t, URIRef):
                    focus_types.add(normalize_type(str(t)))
        relevant = not focus_types or bool(focus_types & allowed_types)
        line = f"{_format_result_path(path)}: {message or 'validation error'} (shape: {source_label})"
        if severity == _SH.Warning or not relevant:
            warnings.append(line)
        else:
            errors.append(line)
        count += 1
        if count >= max_items:
            break
    return errors, warnings


def normalize_jsonld(
    data: dict[str, Any] | list[Any],
    dataset_uri: str,
    url: str,
    target_type: str | None,
    *,
    embed_nodes: bool = True,
) -> dict[str, Any]:
    data = _normalize_iri_suffixes(data)
    nodes = _collect_jsonld_nodes(data)
    if not nodes:
        raise UsageError("No JSON-LD nodes produced by morph-kgc.")

    target = normalize_type(target_type) if target_type else None
    main_node: dict[str, Any] | None = None
    main_old_id = "_:b0"
    for idx, node in enumerate(nodes):
        node_type = _extract_type(node)
        if target and node_type == target:
            main_node = node
            main_old_id = str(node.get("@id") or f"_:b{idx}")
            break
    if main_node is None:
        main_node = nodes[0]
        main_old_id = str(main_node.get("@id") or "_:b0")

    id_map = _build_id_map(nodes, dataset_uri, url)
    node_map: dict[str, dict[str, Any]] = {}
    for idx, node in enumerate(nodes):
        old_id = str(node.get("@id") or f"_:b{idx}")
        new_id = id_map[old_id]
        node["@id"] = new_id
        node_map[new_id] = node

    for node in nodes:
        for key, value in list(node.items()):
            if key in ("@id", "@type"):
                continue
            node[key] = _rewrite_refs(value, id_map, node_map, embed_nodes=embed_nodes)

    main_id = id_map.get(main_old_id)
    if not main_id:
        raise UsageError("Failed to resolve main node @id.")
    if embed_nodes:
        main = node_map[main_id]
        main["@context"] = _SCHEMA_BASE
        blank_nodes = _blank_node_errors(main)
        if blank_nodes:
            raise UsageError("Blank nodes are not allowed in JSON-LD output.")
        return main
    for node in node_map.values():
        node.setdefault("@context", _SCHEMA_BASE)
    graph = {"@context": _SCHEMA_BASE, "@graph": list(node_map.values())}
    blank_nodes = _blank_node_errors(graph)
    if blank_nodes:
        raise UsageError("Blank nodes are not allowed in JSON-LD output.")
    return graph


def generate_from_agent(
    url: str,
    html: str,
    xhtml: str,
    cleaned_xhtml: str,
    api_key: str,
    dataset_uri: str,
    target_type: str | None,
    workdir: Path,
    debug: bool = False,
    max_retries: int = 2,
    max_nesting_depth: int = 2,
    quality_check: bool = True,
    log: Callable[[str], None] | None = None,
) -> tuple[str, dict[str, Any]]:
    debug_path = workdir / "agent_debug.json" if debug else None
    target_name = normalize_type(target_type or "Thing")
    property_guides = property_guides_with_related(target_name, max_nesting_depth)
    allowed_properties = _google_allowed_properties(property_guides)
    allowed_property_set = _mapping_allowed_property_set(property_guides)
    workdir.mkdir(parents=True, exist_ok=True)
    requirements_path = workdir / "requirements.json"
    requirements_path.write_text(
        json.dumps(
            {
                "target_type": target_name,
                "max_depth": max_nesting_depth,
                "types": property_guides,
                "allowed_properties": allowed_properties,
            },
            indent=2,
        )
    )
    html_path = (workdir / "rendered.html").resolve()
    html_path.write_text(html)
    xhtml_path = (workdir / "page.xhtml").resolve()
    xhtml_path.write_text(xhtml)
    cleaned_path = (workdir / "page.cleaned.xhtml").resolve()
    cleaned_path.write_text(cleaned_xhtml)

    shape_specs = shape_specs_for_types(list(property_guides.keys()))
    mapping_validation_path = workdir / "mapping.validation.json"
    mapping_jsonld_path = workdir / "mapping.jsonld"

    yarrml = ""
    mappings: list[dict[str, Any]] = []
    missing_required: list[str] = []
    previous_yarrml: str | None = None
    validation_errors: list[str] | None = None
    validation_report: list[str] | None = None
    missing_recommended: list[str] = []
    xpath_warnings: list[str] = []
    quality_feedback: list[str] | None = None
    quality_score: int | None = None
    jsonld_raw: dict[str, Any] | list[Any] | None = None
    normalized_jsonld: dict[str, Any] | None = None

    for attempt in range(max_retries + 1):
        yarrml = ask_agent_for_yarrml(
            api_key,
            url,
            cleaned_xhtml,
            target_type,
            debug=debug,
            debug_path=debug_path,
            property_guides=property_guides,
            missing_required=missing_required if attempt > 0 else None,
            missing_recommended=missing_recommended if attempt > 0 else None,
            previous_yarrml=previous_yarrml if attempt > 0 else None,
            validation_errors=validation_errors if attempt > 0 else None,
            validation_report=validation_report if attempt > 0 else None,
            xpath_warnings=xpath_warnings if attempt > 0 else None,
            allow_properties=allowed_properties,
            quality_feedback=quality_feedback if attempt > 0 else None,
        )

        yarrml, mappings = _normalize_agent_yarrml(
            yarrml,
            url,
            cleaned_path.as_posix(),
            target_type,
        )
        yarrml_path = workdir / "mapping.yarrml"
        rml_path = workdir / "mapping.ttl"
        yarrml_path.write_text(yarrml)

        try:
            _run_yarrrml_parser(yarrml_path, rml_path)
            _ensure_subject_termtype_iri(rml_path)
            _normalize_reference_formulation(rml_path)
            jsonld_raw = _materialize_jsonld(rml_path)
            jsonld_raw = _fill_jsonld_from_mappings(jsonld_raw, mappings, cleaned_xhtml)
            _ensure_node_ids(jsonld_raw, dataset_uri, url)
            mapping_jsonld_path.write_text(json.dumps(jsonld_raw, indent=2))
            normalized_jsonld = postprocess_jsonld(
                jsonld_raw,
                mappings,
                cleaned_xhtml,
                dataset_uri,
                url,
                target_type=target_type,
            )
            final_jsonld_path = workdir / "structured-data.jsonld"
            final_jsonld_path.write_text(json.dumps(normalized_jsonld, indent=2))
            validation_result = validate_file(str(final_jsonld_path), shape_specs=shape_specs)
            errors, warnings = _validation_messages_for_types(
                validation_result,
                set(property_guides.keys()),
            )
            validation_errors = errors or None
            validation_report = validation_result.report_text.splitlines() if validation_result else None
        except Exception as exc:
            mapping_validation_path.write_text(
                json.dumps(
                    {
                        "conforms": False,
                        "warning_count": 0,
                        "errors": [str(exc)],
                        "warnings": [],
                    },
                    indent=2,
                )
            )
            validation_errors = [str(exc)]
            validation_report = None
            previous_yarrml = yarrml
            continue

        mapped_props = _main_mapping_props(mappings)
        required_props = property_guides.get(target_name, {}).get("required", [])
        recommended_props = property_guides.get(target_name, {}).get("recommended", [])
        missing_required = _missing_required_props(required_props, mapped_props)
        missing_recommended = _missing_recommended_props(recommended_props, mapped_props)
        mapping_errors: list[str] = []
        mapping_errors.extend(_mapping_violations(mappings, allowed_property_set, target_name))
        evidence_warnings = _xpath_evidence_errors(mappings, cleaned_xhtml)
        reusability_warnings = _xpath_reusability_warnings(mappings)
        expected_types = {
            "reviewRating": ("Rating",),
            "author": ("Person", "Organization"),
        }
        mapping_errors.extend(_mapping_type_sanity(mappings, expected_types))
        if reusability_warnings:
            mapping_errors.extend(reusability_warnings)
        warnings_out: list[str] = list(warnings) if "warnings" in locals() else []
        if evidence_warnings:
            warnings_out.extend(evidence_warnings)
        if reusability_warnings:
            warnings_out.extend(reusability_warnings)
        if missing_required:
            warnings_out.append(f"Missing required properties: {', '.join(missing_required)}")
        if mapping_errors:
            mapping_validation_path.write_text(
                json.dumps(
                    {
                        "conforms": False,
                        "warning_count": validation_result.warning_count if "validation_result" in locals() else 0,
                        "errors": mapping_errors,
                        "warnings": warnings_out,
                        "shacl_errors": errors if "errors" in locals() else [],
                    },
                    indent=2,
                )
            )
            validation_errors = mapping_errors
            validation_report = None
            xpath_warnings = warnings_out
        else:
            if _review_rating_dropped(jsonld_raw, mappings, target_type):
                warnings_out.append("Review ratingValue missing; reviewRating dropped.")
            mapping_validation_path.write_text(
                json.dumps(
                    {
                        "conforms": validation_result.conforms if "validation_result" in locals() else True,
                        "warning_count": validation_result.warning_count if "validation_result" in locals() else 0,
                        "errors": errors if "errors" in locals() else [],
                        "warnings": warnings_out,
                    },
                    indent=2,
                )
            )
            validation_errors = errors or None
            validation_report = validation_result.report_text.splitlines() if validation_result else None
            xpath_warnings = warnings_out
            if quality_check:
                quality_score = None
                quality_feedback = None
                try:
                    quality_payload = ask_agent_for_quality(
                        api_key,
                        url,
                        cleaned_xhtml,
                        normalized_jsonld,
                        property_guides,
                        target_type,
                    )
                except ExternalServiceError:
                    quality_payload = None
                if isinstance(quality_payload, dict):
                    score = quality_payload.get("score")
                    if isinstance(score, (int, float)):
                        quality_score = int(score)
                    missing = quality_payload.get("missing_in_jsonld")
                    notes = quality_payload.get("notes")
                    suggested = quality_payload.get("suggested_xpath")
                    feedback: list[str] = []
                    if isinstance(missing, list) and missing:
                        feedback.append("Missing in JSON-LD (present in XHTML):")
                        feedback.extend([str(item) for item in missing])
                    if isinstance(suggested, dict) and suggested:
                        feedback.append("Suggested XPath for missing properties:")
                        for key, value in suggested.items():
                            feedback.append(f"- {key}: {value}")
                    if isinstance(notes, list) and notes:
                        feedback.append("Notes:")
                        feedback.extend([str(item) for item in notes])
                    if feedback:
                        feedback.append(
                            f"Quality score: {quality_score}"
                            if quality_score is not None
                            else "Quality score unavailable"
                        )
                        quality_feedback = feedback
        if validation_errors is None and (not missing_required or attempt >= max_retries):
            if not quality_check or quality_score is None or quality_score >= 7 or attempt >= max_retries:
                break
        previous_yarrml = yarrml

    if jsonld_raw is None:
        raise UsageError("Failed to produce JSON-LD from the generated YARRRML mapping.")
    if validation_errors:
        logger = logging.getLogger("worai")
        logger.warning(
            "YARRRML mapping failed validation after retries; proceeding anyway. "
            f"See {mapping_validation_path} for details."
        )

    if normalized_jsonld is None:
        normalized_jsonld = normalize_jsonld(jsonld_raw, dataset_uri, url, target_type, embed_nodes=False)
    jsonld = normalized_jsonld
    return yarrml, jsonld
