import re
import base64
import hashlib
import warnings
import yaml
from argparse import Namespace
from copy import deepcopy, copy
from typing import Dict, List, Sequence, Tuple, Type, Any, cast, Iterable, Optional, Union

from dlt.common.pendulum import pendulum
from dlt.common.time import ensure_pendulum_datetime_utc
from dlt.common import logger
from dlt.common.json import json
from dlt.common.data_types import TDataType
from dlt.common.exceptions import DictValidationException, ValueErrorWithKnownValues
from dlt.common.normalizers.naming import NamingConvention
from dlt.common.normalizers.naming.snake_case import NamingConvention as SnakeCase
from dlt.common.typing import DictStrAny, REPattern
from dlt.common.validation import TCustomValidator, validate_dict_ignoring_xkeys
from dlt.common.schema import detections
from dlt.common.schema.typing import (
    C_DLT_ID,
    C_DLT_LOAD_ID,
    DLT_NAME_PREFIX,
    C_DLT_LOADS_TABLE_LOAD_ID,
    C_CHILD_PARENT_REF_LABEL,
    C_DESCENDANT_ROOT_REF_LABEL,
    C_ROOT_LOAD_REF_LABEL,
    C_VERSION_SCHEMA_NAME_LABEL,
    C_VERSION_SCHEMA_VERSION_LABEL,
    SCHEMA_ENGINE_VERSION,
    LOADS_TABLE_NAME,
    SIMPLE_REGEX_PREFIX,
    VERSION_TABLE_NAME,
    PIPELINE_STATE_TABLE_NAME,
    ColumnPropInfos,
    TColumnName,
    TFileFormat,
    TPartialTableSchema,
    TSchemaTables,
    TSchemaUpdate,
    TSimpleRegex,
    TStoredSchema,
    TTableProcessingHints,
    TColumnProcessingHints,
    TTableReferenceParam,
    TTableSchema,
    TColumnSchemaBase,
    TColumnSchema,
    TColumnProp,
    TTableFormat,
    TColumnDefaultHint,
    TTableSchemaColumns,
    TTypeDetectionFunc,
    TTypeDetections,
    TWriteDisposition,
    TLoaderMergeStrategy,
    TSchemaContract,
    TSortOrder,
    TTableReference,
)
from dlt.common.schema.exceptions import (
    CannotCoerceColumnException,
    ParentTableNotFoundException,
    TablePropertiesConflictException,
    InvalidSchemaName,
)
from dlt.common.warnings import Dlt100DeprecationWarning


RE_NON_ALPHANUMERIC_UNDERSCORE = re.compile(r"[^a-zA-Z\d_]")
DEFAULT_WRITE_DISPOSITION: TWriteDisposition = "append"


def is_valid_schema_name(name: str) -> bool:
    """Schema name must be a valid python identifier and have max len of 64"""
    return (
        name is not None
        and name.isidentifier()
        and len(name) <= InvalidSchemaName.MAXIMUM_SCHEMA_NAME_LENGTH
    )


def is_nested_table(table: TTableSchema) -> bool:
    """Checks if table is a dlt nested table: connected to parent table via row_key - parent_key reference"""
    # "parent" table hint indicates NESTED table.
    return bool(table.get("parent"))


def may_be_nested(table: TTableSchema) -> bool:
    """Table may be nested if it does not define any primary/merge keys"""
    pks = get_columns_names_with_prop(table, "primary_key", include_incomplete=True)
    mks = get_columns_names_with_prop(table, "merge_key", include_incomplete=True)
    return not pks and not mks


def normalize_schema_name(name: str) -> str:
    """Normalizes schema name by using snake case naming convention. The maximum length is 64 characters"""
    snake_case = SnakeCase(InvalidSchemaName.MAXIMUM_SCHEMA_NAME_LENGTH)
    return snake_case.normalize_identifier(name)


def apply_defaults(stored_schema: TStoredSchema) -> TStoredSchema:
    """Applies default hint values to `stored_schema` in place

    Updates only complete column hints, incomplete columns are preserved intact
    """
    for table_name, table in stored_schema["tables"].items():
        # overwrite name
        table["name"] = table_name
        for column_name in table["columns"]:
            # add default hints to tables
            column = table["columns"][column_name]
            # overwrite column name
            column["name"] = column_name
            # set column with default
            # table["columns"][column_name] = column
        # add default write disposition to root tables
        if not is_nested_table(table):
            if table.get("write_disposition") is None:
                table["write_disposition"] = DEFAULT_WRITE_DISPOSITION
            if table.get("resource") is None:
                table["resource"] = table_name
    return stored_schema


def remove_defaults(stored_schema: TStoredSchema) -> TStoredSchema:
    """Removes default values from `stored_schema` in place, returns the input for chaining

    * removes column and table names from the value
    * removed resource name if same as table name
    """
    clean_tables = deepcopy(stored_schema["tables"])
    for table in clean_tables.values():
        del table["name"]
        # if t.get("resource") == table_name:
        #     del t["resource"]
        for c in table["columns"].values():
            # remove defaults only on complete columns
            # if is_complete_column(c):
            #     remove_column_defaults(c)
            #     # restore "nullable" True because we want to have explicit nullability in stored schema
            #     c["nullable"] = c.get("nullable", True)
            # do not save names
            del c["name"]

    stored_schema["tables"] = clean_tables
    return stored_schema


def has_default_column_prop_value(prop: str, value: Any) -> bool:
    """Checks if `value` is a default for `prop`."""
    # remove all boolean hints that are False, except "nullable" which is removed when it is True
    if prop in ColumnPropInfos:
        return value in ColumnPropInfos[prop].defaults
    # for any unknown hint ie. "x-" the defaults are
    return value in (None, False)


def is_compound_prop(prop: str) -> bool:
    """Checks if a column property is compound."""
    if prop in ColumnPropInfos:
        return ColumnPropInfos[prop].compound is True
    # for any unknown property
    return False


def remove_compound_props(
    columns: TTableSchemaColumns, compound_props: set[str]
) -> TTableSchemaColumns:
    """Removes compound properties from all columns in place.

    Args:
        columns: Table columns to modify.
        compound_props: Set of property names to remove.

    Returns:
        The modified columns dict (same object that was passed in).

    Note: This is a generic property remover, but the name reflects its intended use.
    It removes properties even if their value is False.
    """
    for column in columns.values():
        for prop in compound_props:
            column.pop(prop, None)  # type: ignore[misc]
    return columns


def remove_column_defaults(column_schema: TColumnSchema) -> TColumnSchema:
    """Removes default values from `column_schema` in place, returns the input for chaining"""
    # remove hints with default values
    for h in list(column_schema.keys()):
        if has_default_column_prop_value(h, column_schema[h]):  # type: ignore
            del column_schema[h]  # type: ignore
    return column_schema


def bump_version_if_modified(stored_schema: TStoredSchema) -> Tuple[int, str, str, List[str]]:
    """Bumps the `stored_schema` version and version hash if content modified, returns (new version, new hash, old hash, 10 last hashes) tuple"""
    hash_ = generate_version_hash(stored_schema)
    previous_hash = stored_schema.get("version_hash")
    previous_version = stored_schema.get("version")
    if not previous_hash:
        # if hash was not set, set it without bumping the version, that's initial schema
        # previous_version may not be None for migrating schemas
        stored_schema["version"] = previous_version or 1
    elif hash_ != previous_hash:
        stored_schema["version"] += 1
        store_prev_hash(stored_schema, previous_hash)

    stored_schema["version_hash"] = hash_
    return stored_schema["version"], hash_, previous_hash, stored_schema["previous_hashes"]


def store_prev_hash(
    stored_schema: TStoredSchema, previous_hash: str, max_history_len: int = 10
) -> None:
    # unshift previous hash to previous_hashes and limit array to 10 entries
    previous_hashes = stored_schema["previous_hashes"]
    if previous_hash not in previous_hashes:
        previous_hashes.insert(0, previous_hash)
        if (sur := len(previous_hashes) - max_history_len) > 0:
            del previous_hashes[-sur:]
        # stored_schema["previous_hashes"] = stored_schema["previous_hashes"][:max_history_len]


def generate_version_hash(stored_schema: TStoredSchema) -> str:
    # generates hash out of stored schema content, excluding the hash itself and version
    schema_copy = copy(stored_schema)
    schema_copy.pop("version")
    schema_copy.pop("version_hash", None)
    schema_copy.pop("imported_version_hash", None)
    schema_copy.pop("previous_hashes", None)
    # ignore order of elements when computing the hash
    content = json.dumpb(schema_copy, sort_keys=True)
    h = hashlib.sha3_256(content)
    # additionally check column order
    table_names = sorted((schema_copy.get("tables") or {}).keys())
    if table_names:
        for tn in table_names:
            t = schema_copy["tables"][tn]
            h.update(tn.encode("utf-8"))
            # add column names to hash in order
            for cn in (t.get("columns") or {}).keys():
                h.update(cn.encode("utf-8"))
    return base64.b64encode(h.digest()).decode("ascii")


def verify_schema_hash(
    loaded_schema_dict: DictStrAny, verifies_if_not_migrated: bool = False
) -> bool:
    # generates content hash and compares with existing
    engine_version: str = loaded_schema_dict.get("engine_version")
    # if upgrade is needed, the hash cannot be compared
    if verifies_if_not_migrated and engine_version != SCHEMA_ENGINE_VERSION:
        return True
    # if hash is present we can assume at least v4 engine version so hash is computable
    stored_schema = cast(TStoredSchema, loaded_schema_dict)
    hash_ = generate_version_hash(stored_schema)
    return hash_ == stored_schema["version_hash"]


def normalize_simple_regex_column(naming: NamingConvention, regex: TSimpleRegex) -> TSimpleRegex:
    """Assumes that regex applies to column name and normalizes it."""

    def _normalize(r_: str) -> str:
        is_exact = len(r_) >= 2 and r_[0] == "^" and r_[-1] == "$"
        if is_exact:
            r_ = r_[1:-1]
        # if this a simple string then normalize it
        if r_ == re.escape(r_):
            r_ = naming.normalize_path(r_)
        if is_exact:
            r_ = "^" + r_ + "$"
        return r_

    if regex.startswith(SIMPLE_REGEX_PREFIX):
        return cast(TSimpleRegex, SIMPLE_REGEX_PREFIX + _normalize(regex[3:]))
    else:
        return cast(TSimpleRegex, _normalize(regex))


def canonical_simple_regex(regex: str) -> TSimpleRegex:
    if regex.startswith(SIMPLE_REGEX_PREFIX):
        return cast(TSimpleRegex, regex)
    else:
        return cast(TSimpleRegex, SIMPLE_REGEX_PREFIX + "^" + regex + "$")


def simple_regex_validator(path: str, pk: str, pv: Any, t: Any) -> bool:
    # custom validator on type TSimpleRegex
    if t is TSimpleRegex:
        if not isinstance(pv, str):
            raise DictValidationException(
                f"field `{pk}` value `{pv}` has invalid type `{type(pv).__name__}` while `str` is"
                " expected",
                path,
                t,
                pk,
                pv,
            )
        if pv.startswith(SIMPLE_REGEX_PREFIX):
            # check if regex
            try:
                re.compile(pv[3:])
            except Exception as e:
                raise DictValidationException(
                    f"field `{pk}` value `{pv[3:]}` does not compile as regex: `{str(e)}`",
                    path,
                    t,
                    pk,
                    pv,
                )
        else:
            if RE_NON_ALPHANUMERIC_UNDERSCORE.match(pv):
                raise DictValidationException(
                    f"field `{pk}` value `{pv}` looks like a regex, please prefix with `re:`",
                    path,
                    t,
                    pk,
                    pv,
                )
        # we know how to validate that type
        return True
    else:
        # don't know how to validate this
        return False


def column_name_validator(naming: NamingConvention) -> TCustomValidator:
    def validator(path: str, pk: str, pv: Any, t: Any) -> bool:
        if t is TColumnName:
            if not isinstance(pv, str):
                raise DictValidationException(
                    f"field `{pk}` value `{pv}` has invalid type `{type(pv).__name__}` while"
                    " `str` is expected",
                    path,
                    t,
                    pk,
                    pv,
                )
            try:
                if naming.normalize_path(pv) != pv:
                    raise DictValidationException(
                        f"field `{pk}` value `{pv}` is not a valid column name", path, t, pk, pv
                    )
            except ValueError:
                raise DictValidationException(
                    f"field `{pk}` value `{pv}` is not a valid column name", path, t, pk, pv
                )
            return True
        else:
            return False

    return validator


def _prepare_simple_regex(r: TSimpleRegex) -> str:
    if r.startswith(SIMPLE_REGEX_PREFIX):
        return r[3:]
    else:
        # exact matches
        return "^" + re.escape(r) + "$"


def compile_simple_regex(r: TSimpleRegex) -> REPattern:
    return re.compile(_prepare_simple_regex(r))


def compile_simple_regexes(r: Iterable[TSimpleRegex]) -> REPattern:
    """Compile multiple patterns as one"""
    pattern = "|".join(f"({_prepare_simple_regex(p)})" for p in r)
    if not pattern:  # Don't create an empty pattern that matches everything
        raise ValueError("Cannot create a regex pattern from empty sequence")
    return re.compile(pattern)


def validate_stored_schema(stored_schema: TStoredSchema) -> None:
    # use lambda to verify only non extra fields
    validate_dict_ignoring_xkeys(
        spec=TStoredSchema, doc=stored_schema, path=".", validator_f=simple_regex_validator
    )
    # check child parent relationships
    for table_name, table in stored_schema["tables"].items():
        parent_table_name = table.get("parent")
        if parent_table_name:
            if parent_table_name not in stored_schema["tables"]:
                raise ParentTableNotFoundException(
                    stored_schema["name"], table_name, parent_table_name
                )


def autodetect_sc_type(detection_fs: Sequence[TTypeDetections], t: Type[Any], v: Any) -> TDataType:
    if detection_fs:
        for detection_fn in detection_fs:
            # the method must exist in the module
            detection_f: TTypeDetectionFunc = getattr(detections, "is_" + detection_fn)
            dt = detection_f(t, v)
            if dt is not None:
                return dt
    return None


def is_complete_column(col: TColumnSchemaBase) -> bool:
    """Returns true if column contains enough data to be created at the destination. Must contain a name and a data type. Other hints have defaults."""
    return bool(col.get("name")) and bool(col.get("data_type"))


def is_nullable_column(col: TColumnSchemaBase) -> bool:
    """Returns true if column is nullable"""
    return col.get("nullable", True)


def find_incomplete_columns(table: TTableSchema) -> Iterable[Tuple[TColumnSchemaBase, bool]]:
    """Yields (column, nullable) for all incomplete columns in `table`"""
    for col in table["columns"].values():
        if not is_complete_column(col):
            yield col, is_nullable_column(col)


def compare_complete_columns(a: TColumnSchema, b: TColumnSchema) -> bool:
    """Compares mandatory fields of complete columns"""
    assert is_complete_column(a)
    assert is_complete_column(b)
    return a["data_type"] == b["data_type"] and a["name"] == b["name"]


def compare_table_references(a: TTableReference, b: TTableReference) -> bool:
    if a["referenced_table"] != b["referenced_table"]:
        return False
    a_col_map = dict(zip(a["columns"], a["referenced_columns"]))
    b_col_map = dict(zip(b["columns"], b["referenced_columns"]))
    return a_col_map == b_col_map


def diff_table_references(
    a: Sequence[TTableReference], b: Sequence[TTableReference]
) -> List[TTableReference]:
    """Return a list of references containing references matched by table:
    * References from `b` that are not in `a`
    * References from `b` that are different from the one in `a`
    """
    a_refs_mapping = {ref["referenced_table"]: ref for ref in a}
    new_refs: List[TTableReference] = []
    for b_ref in b:
        table_name = b_ref["referenced_table"]
        if table_name not in a_refs_mapping:
            new_refs.append(b_ref)
        elif not compare_table_references(a_refs_mapping[table_name], b_ref):
            new_refs.append(b_ref)
    return new_refs


def merge_column(
    col_a: TColumnSchema, col_b: TColumnSchema, merge_defaults: bool = True
) -> TColumnSchema:
    """Merges properties from `col_b` into `col_a`, modifying `col_a` in place.

    All properties from `col_b` are copied into `col_a`, potentially overwriting existing values.

    Args:
        col_a: Target column schema that will be modified
        col_b: Source column schema with properties to merge in
        merge_defaults: If False, removes properties with default values from `col_b` before merging.
            This prevents unnecessary default values from being explicitly set in `col_a`.

    Returns:
        The modified col_a (same object that was passed in)
    """
    col_b_clean = col_b if merge_defaults else remove_column_defaults(copy(col_b))
    for n, v in col_b_clean.items():
        col_a[n] = v  # type: ignore

    return col_a


def _collect_and_remove_compound_props(
    source_columns: TTableSchemaColumns,
    target_columns: TTableSchemaColumns,
) -> None:
    """Finds all non-default compound properties in source columns and removes
    those properties from all target columns.

    Args:
        source_columns: Columns to collect compound properties from
        target_columns: Columns to remove compound properties from
    """
    compound_props: set[str] = set()
    for column in source_columns.values():
        compound_props.update(
            prop
            for prop in column
            if is_compound_prop(prop) and not has_default_column_prop_value(prop, column[prop])  # type: ignore[literal-required]
        )
    if compound_props:
        remove_compound_props(columns=target_columns, compound_props=compound_props)


def merge_columns(
    columns_a: TTableSchemaColumns,
    columns_b: TTableSchemaColumns,
    merge_compound_props: bool = True,
) -> TTableSchemaColumns:
    """Merges columns from `columns_b` into `columns_a`, modifying `columns_a` in place.

    For each column in `columns_b`:
    - If column doesn't exist in `columns_a`, it's added
    - If column exists in `columns_a`, properties from `columns_b` are merged into it

    Args:
        columns_a: Target columns dict that will be modified
        columns_b: Source columns dict with columns/properties to merge in
        merge_compound_props: If set to True, compound properties from `columns_b` are merged to `columns_a`,
            If False, compound properties like primary_key and merge_key are replaced entirely
            rather than merged, so `columns_b`'s non-default values fully override `columns_a`'s.

    Returns:
        The modified `columns_a` (same object that was passed in)

    NOTE: Incomplete columns in `columns_a` that become complete in `columns_b` are removed and
    re-added to preserve order.
    """
    if not merge_compound_props:
        _collect_and_remove_compound_props(columns_b, columns_a)

    # remove incomplete columns in table that are complete in diff table
    for col_name, column_b in columns_b.items():
        column_a = columns_a.get(col_name)
        if is_complete_column(column_b):
            if column_a and not is_complete_column(column_a):
                columns_a.pop(col_name)
        if column_a:
            column_b = merge_column(column_a, column_b)
        # set new or updated column
        columns_a[col_name] = column_b
    return columns_a


def ensure_compatible_tables(
    schema_name: str, tab_a: TTableSchema, tab_b: TPartialTableSchema, ensure_columns: bool = True
) -> None:
    """Ensures that `tab_a` and `tab_b` can be merged without conflicts. Conflicts are detected when

    - tables have different names
    - nested tables have different parents
    - tables have any column with incompatible types

    Note: all the identifiers must be already normalized

    """
    if tab_a["name"] != tab_b["name"]:
        raise TablePropertiesConflictException(
            schema_name, tab_a["name"], "name", tab_a["name"], tab_b["name"]
        )
    table_name = tab_a["name"]
    # check if table properties can be merged
    if tab_a.get("parent") != tab_b.get("parent"):
        raise TablePropertiesConflictException(
            schema_name, table_name, "parent", tab_a.get("parent"), tab_b.get("parent")
        )

    if not ensure_columns:
        return

    tab_a_columns = tab_a["columns"]
    for col_b_name, col_b in tab_b["columns"].items():
        if col_b_name in tab_a_columns:
            col_a = tab_a_columns[col_b_name]
            # we do not support changing data types of columns
            if is_complete_column(col_a) and is_complete_column(col_b):
                if not compare_complete_columns(tab_a_columns[col_b_name], col_b):
                    # attempt to update to incompatible columns
                    raise CannotCoerceColumnException(
                        schema_name,
                        table_name,
                        col_b_name,
                        col_b["data_type"],
                        tab_a_columns[col_b_name]["data_type"],
                        None,
                    )


# def compare_tables(tab_a: TTableSchema, tab_b: TTableSchema) -> bool:
#     try:
#         table_name = tab_a["name"]
#         if table_name != tab_b["name"]:
#             raise TablePropertiesConflictException(table_name, "name", table_name, tab_b["name"])
#         diff_table = diff_tables(tab_a, tab_b, ignore_table_name=False)
#         # columns cannot differ
#         return len(diff_table["columns"]) == 0
#     except SchemaException:
#         return False


def merge_table(
    schema_name: str,
    table: TTableSchema,
    partial_table: TPartialTableSchema,
    merge_compound_props: bool = True,
) -> TPartialTableSchema:
    """Merges `partial_table` into `table` in place.

    Merge rules:
    - New columns from `partial_table` are added to `table`
    - Existing columns are updated: properties from `partial_table` overwrite `table`
    - Incomplete columns in `table` that become complete in `partial_table` are removed
      and re-added at the end to preserve column order
    - References are merged by `referenced_table`: new refs are added, existing refs
      for the same table are updated
    - Other table-level properties from `partial_table` overwrite those in `table`
    - Nothing is deleted from `table` (columns, properties) unless explicitly overwritten

    Args:
        schema_name: Name of the schema, used for error messages.
        table: Target table schema that will be modified in place.
        partial_table: Source table schema with columns/properties to merge in.
        merge_compound_props: If True (default), compound properties (primary_key,
            merge_key, partition, cluster) from `partial_table` are merged additively
            with those in `table`. If False, compound properties in `partial_table`
            are authoritative: any compound property present in `table` but absent
            in `partial_table` is removed.

    Returns:
        The `partial_table` argument (unmodified).

    Raises:
        TablePropertiesConflictException: If tables have different names, different
            parents, or if a nested table has a resource property set.
    """
    ensure_compatible_tables(schema_name, table, partial_table, ensure_columns=False)

    # nested tables with the resource property set - this should not really happen
    if is_nested_table(table) and (resource := partial_table.get("resource")):
        raise TablePropertiesConflictException(
            schema_name, table["name"], "resource", resource, table.get("parent")
        )

    updated_columns = merge_columns(
        table["columns"], partial_table["columns"], merge_compound_props
    )

    # merge references: add new/changed refs from partial_table to table
    if "references" in partial_table:
        new_refs = diff_table_references(
            table.get("references", []), partial_table.get("references", [])
        )
        if new_refs:
            existing_refs = {ref["referenced_table"]: ref for ref in table.get("references", [])}
            for ref in new_refs:
                existing_refs[ref["referenced_table"]] = ref
            table["references"] = list(existing_refs.values())

    # apply table-level hints that are new or changed (skip columns, references, unchanged values)
    for k, v in partial_table.items():
        if k in ("columns", "references"):
            continue
        if table.get(k) != v:
            table[k] = v  # type: ignore[literal-required]
    table["columns"] = updated_columns

    return partial_table


def normalize_table_identifiers(table: TTableSchema, naming: NamingConvention) -> TTableSchema:
    """Normalizes all table and column names in `table` schema according to current schema naming convention and returns
    new instance with modified table schema.

    Naming convention like snake_case may produce name collisions with the column names. Colliding column schemas are merged
    where the column that is defined later in the dictionary overrides earlier column.

    Note that resource name is not normalized.
    """

    table = copy(table)
    table["name"] = naming.normalize_tables_path(table["name"])
    parent = table.get("parent")
    if parent:
        table["parent"] = naming.normalize_tables_path(parent)
    columns = table.get("columns")
    if columns:
        new_columns: TTableSchemaColumns = {}
        for c in columns.values():
            c = copy(c)
            origin_c_name = c["name"]
            new_col_name = c["name"] = naming.normalize_path(c["name"])
            # re-index columns as the name changed, if name space was reduced then
            # some columns now collide with each other. so make sure that we merge columns that are already there
            if new_col_name in new_columns:
                new_columns[new_col_name] = merge_column(
                    new_columns[new_col_name], c, merge_defaults=False
                )
                logger.warning(
                    f"In schema {naming} column {origin_c_name} got normalized into"
                    f" {new_col_name} which collides with other column. Both columns got merged"
                    " into one."
                )
            else:
                new_columns[new_col_name] = c
        table["columns"] = new_columns
    references = table.get("references")
    if references:
        new_references = {}
        for ref in references:
            new_ref = copy(ref)
            new_ref["referenced_table"] = naming.normalize_tables_path(ref["referenced_table"])
            new_ref["columns"] = [naming.normalize_path(c) for c in ref["columns"]]
            new_ref["referenced_columns"] = [
                naming.normalize_path(c) for c in ref["referenced_columns"]
            ]
            if new_ref["referenced_table"] in new_references:
                logger.warning(
                    f"In schema {naming} table {table['name']} has multiple references to"
                    f" {new_ref['referenced_table']}. Only the last one is preserved."
                )
            new_references[new_ref["referenced_table"]] = new_ref

        table["references"] = list(new_references.values())
    return table


def has_table_seen_data(table: TTableSchema) -> bool:
    """Checks if normalizer has seen data coming to the table."""
    return "x-normalizer" in table and table["x-normalizer"].get("seen-data", None) is True


def remove_processing_hints(tables: TSchemaTables) -> TSchemaTables:
    """
    Removes processing hints like x-normalizer and x-loader from schema tables and columns.
    Modifies the input tables and returns it for convenience.
    """
    table_hints, col_hints = get_processing_hints(tables)

    # Remove table-level hints
    for table_name, hints in table_hints.items():
        for hint in hints:
            tables[table_name].pop(hint, None)  # type: ignore[misc]

    # Remove column-level hints
    for table_name, cols in col_hints.items():
        for col_name, hints in cols.items():
            for hint in hints:
                tables[table_name]["columns"][col_name].pop(hint, None)  # type: ignore[misc]

    return tables


def has_seen_null_first_hint(column_schema: TColumnSchema) -> bool:
    """Checks if `column_schema` has seen seen-null-first hint set to True in the x-normalizer hints."""
    return bool(column_schema.get("x-normalizer", {}).get("seen-null-first"))


def remove_seen_null_first_hint(column_schema: TColumnSchema) -> TColumnSchema:
    """Removes seen-null-first hint from the x-normalizer hints in `column_schema` in place,
    if the x-normalizer section becomes empty after removing the hint, it is also removed, returns the modified input
    """
    x_normalizer = column_schema.setdefault("x-normalizer", {})
    if x_normalizer.get("seen-null-first"):
        x_normalizer.pop("seen-null-first", None)
    if not x_normalizer:
        column_schema.pop("x-normalizer", None)
    return column_schema


def get_processing_hints(
    tables: TSchemaTables,
) -> Tuple[Dict[str, List[str]], Dict[str, Dict[str, List[str]]]]:
    """
    Finds processing hints from schema tables and columns.

    Returns:
        A tuple containing:
            - A dictionary mapping table names to a list of table-level processing hints (e.g., 'x-normalizer', 'x-loader').
            - A dictionary mapping table names to another dictionary that maps column names to a list of column-level processing hints (e.g., 'x-normalizer').
    """
    table_hints: Dict[str, List[str]] = {}
    col_hints: Dict[str, Dict[str, List[str]]] = {}

    for table in tables.values():  # <- This is the correct line
        table_name = table["name"]

        for hint in TTableProcessingHints.__annotations__.keys():
            if hint in table:
                table_hints.setdefault(table_name, []).append(hint)

        for col_name, col in table["columns"].items():
            for hint in TColumnProcessingHints.__annotations__.keys():
                if hint in col:
                    col_hints.setdefault(table_name, {}).setdefault(col_name, []).append(hint)

    return table_hints, col_hints


def hint_to_column_prop(h: TColumnDefaultHint) -> TColumnProp:
    if h == "not_null":
        return "nullable"
    return h


def get_columns_names_with_prop(
    table: TTableSchema, column_prop: Union[TColumnProp, str], include_incomplete: bool = False
) -> List[str]:
    return [
        c_n
        for c_n, c in table["columns"].items()
        if column_prop in c
        and not has_default_column_prop_value(column_prop, c[column_prop])  # type: ignore[literal-required]
        and (include_incomplete or is_complete_column(c))
    ]


def get_first_column_name_with_prop(
    table: TTableSchema, column_prop: Union[TColumnProp, str], include_incomplete: bool = False
) -> Optional[str]:
    """Returns name of first column in `table` schema with property `column_prop` or None if no such column exists."""
    column_names = get_columns_names_with_prop(table, column_prop, include_incomplete)
    if len(column_names) > 0:
        return column_names[0]
    return None


def has_column_with_prop(
    table: TTableSchema, column_prop: Union[TColumnProp, str], include_incomplete: bool = False
) -> bool:
    """Checks if `table` schema contains column with property `column_prop`."""
    return len(get_columns_names_with_prop(table, column_prop, include_incomplete)) > 0


def get_dedup_sort_tuple(
    table: TTableSchema, include_incomplete: bool = False
) -> Optional[Tuple[str, TSortOrder]]:
    """Returns tuple with dedup sort information.

    First element is the sort column name, second element is the sort order.

    Returns None if "dedup_sort" hint was not provided.
    """
    dedup_sort_col = get_first_column_name_with_prop(table, "dedup_sort", include_incomplete)
    if dedup_sort_col is None:
        return None
    dedup_sort_order = table["columns"][dedup_sort_col]["dedup_sort"]
    return (dedup_sort_col, dedup_sort_order)


def get_validity_column_names(table: TTableSchema) -> List[Optional[str]]:
    return [
        get_first_column_name_with_prop(table, "x-valid-from"),
        get_first_column_name_with_prop(table, "x-valid-to"),
    ]


def get_active_record_timestamp(table: TTableSchema) -> Optional[pendulum.DateTime]:
    # method assumes a column with "x-active-record-timestamp" property exists
    cname = get_first_column_name_with_prop(table, "x-active-record-timestamp")
    hint_val = table["columns"][cname]["x-active-record-timestamp"]  # type: ignore[typeddict-item]
    return None if hint_val is None else ensure_pendulum_datetime_utc(hint_val)


def merge_schema_updates(schema_updates: Sequence[TSchemaUpdate]) -> TSchemaTables:
    aggregated_update: TSchemaTables = {}
    for schema_update in schema_updates:
        for table_name, table_updates in schema_update.items():
            for partial_table in table_updates:
                # aggregate schema updates
                aggregated_table = aggregated_update.setdefault(table_name, partial_table)
                aggregated_table["columns"].update(partial_table["columns"])
    return aggregated_update


def get_inherited_table_hint(
    tables: TSchemaTables, table_name: str, table_hint_name: str, allow_none: bool = False
) -> Any:
    table = tables.get(table_name, {})
    if table_hint_name in table:
        return table[table_hint_name]  # type: ignore[literal-required]

    if is_nested_table(table):
        return get_inherited_table_hint(tables, table.get("parent"), table_hint_name, allow_none)

    if allow_none:
        return None

    raise ValueError(
        f"No table hint `{table_hint_name}` found in the chain of tables for table `{table_name}`."
    )


def get_write_disposition(tables: TSchemaTables, table_name: str) -> TWriteDisposition:
    """Returns table hint of a table if present. If not, looks up into parent table"""
    return cast(
        TWriteDisposition,
        get_inherited_table_hint(tables, table_name, "write_disposition", allow_none=False),
    )


def get_table_format(tables: TSchemaTables, table_name: str) -> TTableFormat:
    return cast(
        TTableFormat, get_inherited_table_hint(tables, table_name, "table_format", allow_none=True)
    )


def get_file_format(tables: TSchemaTables, table_name: str) -> TFileFormat:
    return cast(
        TFileFormat, get_inherited_table_hint(tables, table_name, "file_format", allow_none=True)
    )


def get_merge_strategy(tables: TSchemaTables, table_name: str) -> TLoaderMergeStrategy:
    return cast(
        TLoaderMergeStrategy,
        get_inherited_table_hint(tables, table_name, "x-merge-strategy", allow_none=True),
    )


def fill_hints_from_parent_and_clone_table(
    tables: TSchemaTables, table: TTableSchema
) -> TTableSchema:
    """Takes write disposition and table format from parent tables if not present"""
    # make a copy of the schema so modifications do not affect the original document
    table = deepcopy(table)
    table_name = table["name"]
    if "write_disposition" not in table:
        table["write_disposition"] = get_write_disposition(tables, table_name)
    if "table_format" not in table:
        if table_format := get_table_format(tables, table_name):
            table["table_format"] = table_format
    if "file_format" not in table:
        if file_format := get_file_format(tables, table_name):
            table["file_format"] = file_format
    if "x-merge-strategy" not in table:
        if strategy := get_merge_strategy(tables, table_name):
            table["x-merge-strategy"] = strategy  # type: ignore[typeddict-unknown-key]
    return table


def table_schema_has_type(table: TTableSchema, _typ: TDataType) -> bool:
    """Checks if `table` schema contains column with type _typ"""
    return any(c.get("data_type") == _typ for c in table["columns"].values())


def table_schema_has_type_with_precision(table: TTableSchema, _typ: TDataType) -> bool:
    """Checks if `table` schema contains column with type _typ and precision set"""
    return any(
        c.get("data_type") == _typ and c.get("precision") is not None
        for c in table["columns"].values()
    )


def get_data_and_dlt_tables(tables: TSchemaTables) -> tuple[list[TTableSchema], list[TTableSchema]]:
    """Separate a list of dlt TTableSchema into a two lists: data tables and internal dlt tables.

    This should be equivalent to `dlt.Schema.data_tables()` and `dlt.Schema.dlt_tables()`
    """
    data_tables: list[TTableSchema] = []
    dlt_tables: list[TTableSchema] = []
    for table_name, table in tables.items():
        if table_name in (VERSION_TABLE_NAME, LOADS_TABLE_NAME, PIPELINE_STATE_TABLE_NAME):
            dlt_tables.append(table)
        else:
            data_tables.append(table)

    return data_tables, dlt_tables


def get_dlt_prefix_by_naming_convetion(naming: NamingConvention) -> str:
    """The dlt prefix, used for tables and columns,normalized according to the naming convention"""
    return naming.normalize_table_identifier(DLT_NAME_PREFIX)


def is_dlt_table_or_column(name: str, normalized_dlt_prefix: str) -> bool:
    """
    Check if a table or column name is a dlt internal name by checking if it starts with dlt prefix.

    Args:
        name: The table or column name to check
        normalized_dlt_prefix: The dlt prefix to check against, normalized by the naming convention
    """
    return name.startswith(normalized_dlt_prefix)


def remove_dlt_columns_from_table(
    table_schema: TTableSchema,
    normalized_dlt_prefix: str,
    exclude_dlt_columns: bool = True,
) -> TTableSchema:
    """
    Remove dlt columns from a single table schema.

    Args:
        table_schema: The table schema to filter
        normalized_dlt_prefix: The dlt prefix to filter by, normalized by the naming convention
        exclude_dlt_columns: If True, remove columns whose name starts with the given prefix

    Returns:
        A new table schema with dlt columns optionally filtered out
    """
    # Create a copy of the table schema, preserving all fields except columns
    new_table_schema = cast(TTableSchema, {k: v for k, v in table_schema.items() if k != "columns"})

    if "columns" in table_schema:
        if exclude_dlt_columns:
            new_table_schema["columns"] = {
                col: col_def
                for col, col_def in table_schema["columns"].items()
                if not is_dlt_table_or_column(col, normalized_dlt_prefix)
            }
        else:
            new_table_schema["columns"] = table_schema["columns"]

    return new_table_schema


def exclude_dlt_entities(
    table_schemas: Iterable[TTableSchema],
    normalized_dlt_prefix: str,
    exclude_dlt_tables: bool = True,
    exclude_dlt_columns: bool = True,
) -> List[TTableSchema]:
    """
    Filter out dlt tables and/or dlt columns from a collection of table schemas.

    Args:
        table_schemas: An iterable of table schemas to filter
        normalized_dlt_prefix: The normalized name of the prefix used to denote internal dlt
            columns and tables, according to the used naming convention
        exclude_dlt_tables: If True, remove tables whose name starts with the given prefix
        exclude_dlt_columns: If True, remove columns whose name starts with the given prefix
    Returns:
        List of filtered table schemas.

    Note: dlt supports changing the default prefix, see schema._dlt_tables_prefix attribute to get
        the source of truth for your schema
    """
    filtered_tables: List[TTableSchema] = []

    for table_schema in table_schemas:
        table_name = table_schema["name"]

        # Skip dlt tables if requested
        if exclude_dlt_tables and is_dlt_table_or_column(table_name, normalized_dlt_prefix):
            continue

        # Remove dlt columns if requested
        filtered_table = remove_dlt_columns_from_table(
            table_schema, normalized_dlt_prefix, exclude_dlt_columns
        )
        filtered_tables.append(filtered_table)

    return filtered_tables


def get_root_table(tables: TSchemaTables, table_name: str) -> TTableSchema:
    """Finds root (without parent) of a `table_name` following the nested references (row_key - parent_key)."""
    table = tables[table_name]
    if is_nested_table(table):
        return get_root_table(tables, table.get("parent"))
    return table


def get_nested_tables(
    tables: TSchemaTables,
    table_name: str,
    max_nesting: Optional[int] = None,
    include_self: Optional[bool] = True,
) -> List[TTableSchema]:
    """Get nested tables for table name and return a list of tables ordered by ancestry so the nested tables are always after their parents

    Note that this function follows only NESTED TABLE reference typically expressed on _dlt_parent_id (PARENT_KEY) to _dlt_id (ROW_KEY).

    Args:
        tables (TSchemaTables): A mapping of table names to their table schema definitions. This is used to look up the root table
            and to recursively find its nested child tables by following their "parent" references.
        table_name (str): The name of the root table from which to collect nested tables.
        max_nesting (Optional[int]): If specified, limits the depth of nesting. 0 = only the root table, 1 = root + direct children, etc.
        include_self (Optional[bool]): If False, the root table itself is excluded from the returned list.

    Returns:
        List[TTableSchema]: A list of nested tables.
    """
    if table_name not in tables:
        return []

    chain: List[TTableSchema] = []

    def _child(t: TTableSchema, current_level: int = 0) -> None:
        name = t["name"]

        if include_self or current_level > 0:
            chain.append(t)

        # Stop recursion if we've reached max nesting level
        if max_nesting is not None and current_level >= max_nesting:
            return

        for candidate in tables.values():
            if is_nested_table(candidate) and candidate.get("parent") == name:
                _child(candidate, current_level + 1)

    _child(tables[table_name])
    return chain


def group_tables_by_resource(
    tables: TSchemaTables, pattern: Optional[REPattern] = None
) -> Dict[str, List[TTableSchema]]:
    """Create a dict of resources and their associated tables and descendant tables
    If `pattern` is supplied, the result is filtered to only resource names matching the pattern.
    """
    result: Dict[str, List[TTableSchema]] = {}
    for table in tables.values():
        resource = table.get("resource")
        if resource and (pattern is None or pattern.match(resource)):
            resource_tables = result.setdefault(resource, [])
            resource_tables.extend(get_nested_tables(tables, table["name"]))
    return result


def create_root_child_reference(tables: TSchemaTables, table_name: str) -> TTableReference:
    """Create a Reference between `{table}.{root_key}` and `{root}.{row_key}`"""
    child_table = tables.get(table_name)
    if child_table is None:
        raise ValueErrorWithKnownValues(
            key="table_name", value_received=table_name, valid_values=list(tables.keys())
        )

    if is_nested_table(child_table) is False:
        raise ValueError(f"Table `{table_name}` is already a root table.")

    root_table = get_root_table(tables, table_name)

    child_root_key: str = get_first_column_name_with_prop(child_table, "root_key")
    if child_root_key is None:
        raise ValueError(
            "No `root_key` found for table `{table_name}`. Set `root_key=True` on"
            " the `@dlt.source` producing this data to generate a `_dlt_root_id` column."
        )
    root_row_key: str = get_first_column_name_with_prop(root_table, "row_key")

    return TTableReference(
        label=C_DESCENDANT_ROOT_REF_LABEL,
        cardinality="many_to_one",
        table=table_name,
        columns=[child_root_key],
        referenced_table=root_table["name"],
        referenced_columns=[root_row_key],
    )


def get_all_root_child_references_from_root(
    tables: TSchemaTables, table_name: str
) -> list[TTableReference]:
    root_table = tables.get(table_name)
    if is_nested_table(root_table) is True:
        raise ValueError(f"Table `{table_name}` is not a root table.")

    children_refs = []
    # skip the first table in chain, which is the root; i.e., the current one
    for child_table in get_nested_tables(tables, table_name)[1:]:
        # try/except because a root table may or may not have child with `root_key` enabled
        try:
            child_ref = create_root_child_reference(tables, child_table["name"])
            children_refs.append(child_ref)
        except ValueError:
            pass

    return children_refs


def create_parent_child_reference(tables: TSchemaTables, table_name: str) -> TTableReference:
    """Create a Reference between `{table}.{parent_key}` and `{parent}.{row_key}`"""
    child_table = tables.get(table_name)
    if child_table is None:
        raise ValueErrorWithKnownValues(
            key="table_name", value_received=table_name, valid_values=list(tables.keys())
        )

    parent_table_name = child_table.get("parent")
    if parent_table_name is None:
        raise ValueError(f"Table `{table_name}` is a root table and has no parent.")
    parent_table = tables.get(parent_table_name)

    child_parent_key: str = get_first_column_name_with_prop(child_table, "parent_key")
    parent_row_key: str = get_first_column_name_with_prop(parent_table, "row_key")

    return TTableReference(
        label=C_CHILD_PARENT_REF_LABEL,
        cardinality="many_to_one",
        table=table_name,
        columns=[child_parent_key],
        referenced_table=parent_table_name,
        referenced_columns=[parent_row_key],
    )


def get_all_parent_child_references_from_root(
    tables: TSchemaTables, table_name: str
) -> list[TTableReference]:
    root_table = tables.get(table_name)
    if is_nested_table(root_table) is True:
        raise ValueError(f"Table `{table_name}` is not a root table.")

    children_refs = []
    # skip the first table in chain, which is the root; i.e., the current one
    for child_table in get_nested_tables(tables, table_name)[1:]:
        # try/except because a root table may or may not have child with `root_key` enabled
        try:
            child_ref = create_parent_child_reference(tables, child_table["name"])
            children_refs.append(child_ref)
        except ValueError:
            pass

    return children_refs


def create_load_table_reference(
    table: TTableSchema, *, naming: NamingConvention = None
) -> TTableReference:
    """Create a Reference between `{table}._dlt_load_id` and `_dlt_loads.load_id`"""
    # TODO temporary solution; refactor caller to always explicitly pass `naming`
    naming = naming if naming else Namespace(normalize_identifier=lambda x: x)  # type: ignore[assignment]

    load_id_column = naming.normalize_identifier(C_DLT_LOAD_ID)
    if table["columns"].get(load_id_column) is None:
        raise ValueError(
            f"Table `{table['name']}` is not a root table and has no `{load_id_column}` column."
        )

    return TTableReference(
        label=C_ROOT_LOAD_REF_LABEL,
        cardinality="many_to_one",
        table=table["name"],
        columns=[load_id_column],
        referenced_table=naming.normalize_identifier(LOADS_TABLE_NAME),
        referenced_columns=[naming.normalize_identifier(C_DLT_LOADS_TABLE_LOAD_ID)],
    )


def create_version_and_loads_hash_reference(
    tables: TSchemaTables, *, naming: NamingConvention = None
) -> TTableReference:
    # TODO temporary solution; refactor caller to always explicitly pass `naming`
    naming = naming if naming else Namespace(normalize_identifier=lambda x: x)  # type: ignore[assignment]

    version_table_name = naming.normalize_identifier(VERSION_TABLE_NAME)
    if version_table_name not in tables:
        raise ValueError(
            f"Table `{version_table_name}` not found in tables: `{list(tables.keys())}`"
        )

    load_table_name = naming.normalize_identifier(LOADS_TABLE_NAME)
    if load_table_name not in tables:
        raise ValueError(f"Table `{load_table_name}` not found in tables: `{list(tables.keys())}`")

    return TTableReference(
        label=C_VERSION_SCHEMA_VERSION_LABEL,
        cardinality="one_to_many",
        table=version_table_name,
        columns=[naming.normalize_identifier("version_hash")],
        referenced_table=load_table_name,
        referenced_columns=[naming.normalize_identifier("schema_version_hash")],
    )


def create_version_and_loads_schema_name_reference(
    tables: TSchemaTables, *, naming: NamingConvention = None
) -> TTableReference:
    # TODO temporary solution; refactor caller to always explicitly pass `naming`
    naming = naming if naming else Namespace(normalize_identifier=lambda x: x)  # type: ignore[assignment]

    version_table_name = naming.normalize_identifier(VERSION_TABLE_NAME)
    if version_table_name not in tables:
        raise ValueError(
            f"Table `{version_table_name}` not found in tables: `{list(tables.keys())}`"
        )

    load_table_name = naming.normalize_identifier(LOADS_TABLE_NAME)
    if load_table_name not in tables:
        raise ValueError(f"Table `{load_table_name}` not found in tables: `{list(tables.keys())}`")

    return TTableReference(
        label=C_VERSION_SCHEMA_NAME_LABEL,
        cardinality="many_to_many",
        table=version_table_name,
        columns=[naming.normalize_identifier("schema_name")],
        referenced_table=load_table_name,
        referenced_columns=[naming.normalize_identifier("schema_name")],
    )


def migrate_complex_types(table: TTableSchema, warn: bool = False) -> None:
    if "columns" not in table:
        return
    table_name = table.get("name")
    for col_name, column in table["columns"].items():
        if data_type := column.get("data_type"):
            if data_type == "complex":
                if warn:
                    warnings.warn(
                        f"`complex` data type found on column {col_name} table {table_name} is"
                        " deprecated. Please use `json` type instead.",
                        Dlt100DeprecationWarning,
                        stacklevel=3,
                    )
                column["data_type"] = "json"


def version_table() -> TTableSchema:
    # NOTE: always add new columns at the end of the table so we have identical layout
    # after an update of existing tables (always at the end)
    # set to nullable so we can migrate existing tables
    # WARNING: do not reorder the columns
    table = new_table(
        VERSION_TABLE_NAME,
        columns=[
            {
                "name": "version",
                "data_type": "bigint",
                "nullable": False,
            },
            {"name": "engine_version", "data_type": "bigint", "nullable": False},
            {"name": "inserted_at", "data_type": "timestamp", "nullable": False},
            {"name": "schema_name", "data_type": "text", "nullable": False},
            {"name": "version_hash", "data_type": "text", "nullable": False},
            {"name": "schema", "data_type": "text", "nullable": False},
        ],
    )
    table["write_disposition"] = "skip"
    table["description"] = "Created by DLT. Tracks schema updates"
    return table


def loads_table() -> TTableSchema:
    # NOTE: always add new columns at the end of the table so we have identical layout
    # after an update of existing tables (always at the end)
    # set to nullable so we can migrate existing tables
    # WARNING: do not reorder the columns
    table = new_table(
        LOADS_TABLE_NAME,
        columns=[
            {
                "name": C_DLT_LOADS_TABLE_LOAD_ID,
                "data_type": "text",
                "nullable": False,
                "precision": 64,
            },
            {"name": "schema_name", "data_type": "text", "nullable": True},
            {"name": "status", "data_type": "bigint", "nullable": False},
            {"name": "inserted_at", "data_type": "timestamp", "nullable": False},
            {
                "name": "schema_version_hash",
                "data_type": "text",
                "nullable": True,
            },
        ],
    )
    table["write_disposition"] = "skip"
    table["description"] = "Created by DLT. Tracks completed loads"
    return table


def dlt_id_column() -> TColumnSchema:
    """Definition of dlt id column"""
    return {
        "name": C_DLT_ID,
        "data_type": "text",
        "nullable": False,
        "unique": True,
        "row_key": True,
        "precision": 64,
    }


def dlt_load_id_column() -> TColumnSchema:
    """Definition of dlt load id column"""
    return {"name": "_dlt_load_id", "data_type": "text", "nullable": False, "precision": 64}


def pipeline_state_table(add_dlt_id: bool = False) -> TTableSchema:
    # NOTE: always add new columns at the end of the table so we have identical layout
    # after an update of existing tables (always at the end)
    # set to nullable so we can migrate existing tables
    # WARNING: do not reorder the columns
    columns: List[TColumnSchema] = [
        {"name": "version", "data_type": "bigint", "nullable": False},
        {"name": "engine_version", "data_type": "bigint", "nullable": False},
        {"name": "pipeline_name", "data_type": "text", "nullable": False},
        {"name": "state", "data_type": "text", "nullable": False},
        {"name": "created_at", "data_type": "timestamp", "nullable": False},
        {"name": "version_hash", "data_type": "text", "nullable": True},
        dlt_load_id_column(),
    ]
    if add_dlt_id:
        columns.append(dlt_id_column())
    table = new_table(
        PIPELINE_STATE_TABLE_NAME,
        write_disposition="append",
        columns=columns,
        # always use caps preferred file format for processing
        file_format="preferred",
    )
    table["description"] = "Created by DLT. Tracks pipeline state"
    return table


def new_table(
    table_name: str,
    parent_table_name: str = None,
    write_disposition: TWriteDisposition = None,
    columns: Sequence[TColumnSchema] = None,
    validate_schema: bool = False,
    resource: str = None,
    schema_contract: TSchemaContract = None,
    table_format: TTableFormat = None,
    file_format: TFileFormat = None,
    references: Sequence[TTableReference] = None,
) -> TTableSchema:
    table: TTableSchema = {
        "name": table_name,
        "columns": {} if columns is None else {c["name"]: c for c in columns},
    }

    if write_disposition:
        table["write_disposition"] = write_disposition
    if schema_contract is not None:
        table["schema_contract"] = schema_contract
    if table_format:
        table["table_format"] = table_format
    if file_format:
        table["file_format"] = file_format
    if references:
        table["references"] = references
    if resource:
        table["resource"] = resource
    if parent_table_name:
        table["parent"] = parent_table_name
    else:
        # set only for root tables
        if not write_disposition:
            # set write disposition only for root tables
            table["write_disposition"] = DEFAULT_WRITE_DISPOSITION
        table["resource"] = resource or table_name

    # migrate complex types to json
    migrate_complex_types(table, warn=True)

    if validate_schema:
        validate_dict_ignoring_xkeys(
            spec=TColumnSchema,
            doc=table["columns"],
            path=f"new_table/{table_name}",
        )
    return table


def new_column(
    column_name: str,
    data_type: TDataType = None,
    nullable: bool = True,
    precision: int = None,
    scale: int = None,
    validate_schema: bool = False,
) -> TColumnSchema:
    column: TColumnSchema = {"name": column_name, "nullable": nullable}
    if data_type:
        column["data_type"] = data_type
    if precision is not None:
        column["precision"] = precision
    if scale is not None:
        column["scale"] = scale
    if validate_schema:
        validate_dict_ignoring_xkeys(
            spec=TColumnSchema,
            doc=column,
            path=f"new_column/{column_name}",
        )

    return column


def default_hints() -> Dict[TColumnDefaultHint, List[TSimpleRegex]]:
    return None


def standard_type_detections() -> List[TTypeDetections]:
    return ["iso_timestamp"]


def to_pretty_json(stored_schema: TStoredSchema) -> str:
    return json.dumps(stored_schema, pretty=True)


def to_pretty_yaml(stored_schema: TStoredSchema) -> str:
    return yaml.dump(stored_schema, allow_unicode=True, default_flow_style=False, sort_keys=False)
