import json
import logging
import os
import time
import uuid
from contextlib import contextmanager
from copy import deepcopy
from dataclasses import dataclass, fields
from errno import ENOENT
from functools import lru_cache
from pathlib import Path
from typing import (
    Any,
    Callable,
    Dict,
    Iterator,
    List,
    Literal,
    Optional,
    Set,
    Tuple,
    Type,
    Union,
)

from recce.event import log_performance
from recce.exceptions import RecceException
from recce.util.cll import CLLPerformanceTracking, cll
from recce.util.lineage import (
    build_column_key,
    filter_dependency_maps,
    find_downstream,
    find_upstream,
)
from recce.util.perf_tracking import LineagePerfTracker
from recce.util.startup_perf import track_timing

from ...tasks.profile import ProfileTask
from ...util.breaking import BreakingPerformanceTracking, parse_change_category

try:
    import agate
    import dbt.adapters.factory
    from dbt.contracts.state import PreviousState
except ImportError as e:
    print("Error: dbt module not found. Please install it by running:")
    print("pip install dbt-core dbt-<adapter>")
    raise e
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer

from recce.adapter.base import BaseAdapter
from recce.state import ArtifactsRoot

from ...models import RunType
from ...models.types import (
    CllColumn,
    CllData,
    CllNode,
    LineageDiff,
    NodeChange,
    NodeDiff,
)
from ...tasks import (
    HistogramDiffTask,
    ProfileDiffTask,
    QueryBaseTask,
    QueryDiffTask,
    QueryTask,
    RowCountDiffTask,
    RowCountTask,
    Task,
    TopKDiffTask,
    ValueDiffDetailTask,
    ValueDiffTask,
)
from .dbt_version import DbtVersion

dbt_supported_registry: Dict[RunType, Type[Task]] = {
    RunType.QUERY: QueryTask,
    RunType.QUERY_BASE: QueryBaseTask,
    RunType.QUERY_DIFF: QueryDiffTask,
    RunType.VALUE_DIFF: ValueDiffTask,
    RunType.VALUE_DIFF_DETAIL: ValueDiffDetailTask,
    RunType.PROFILE: ProfileTask,
    RunType.PROFILE_DIFF: ProfileDiffTask,
    RunType.ROW_COUNT: RowCountTask,
    RunType.ROW_COUNT_DIFF: RowCountDiffTask,
    RunType.TOP_K_DIFF: TopKDiffTask,
    RunType.HISTOGRAM_DIFF: HistogramDiffTask,
}

# Reference: https://github.com/AltimateAI/vscode-dbt-power-user/blob/master/dbt_core_integration.py

get_adapter_orig = dbt.adapters.factory.get_adapter


def get_adapter(config):
    if hasattr(config, "adapter"):
        return config.adapter
    else:
        return get_adapter_orig(config)


dbt.adapters.factory.get_adapter = get_adapter

# All dbt import should after overwriting the get_adapter
from dbt.adapters.base import Column  # noqa: E402
from dbt.adapters.factory import get_adapter_class_by_name  # noqa: E402
from dbt.adapters.sql import SQLAdapter  # noqa: E402
from dbt.config.runtime import RuntimeConfig  # noqa: E402
from dbt.contracts.graph.manifest import (  # noqa: E402
    MacroManifest,
    Manifest,
    ManifestMetadata,
    WritableManifest,
)
from dbt.contracts.graph.nodes import ManifestNode  # noqa: E402
from dbt.contracts.results import CatalogArtifact  # noqa: E402
from dbt.flags import set_from_args  # noqa: E402
from dbt.parser.manifest import process_node  # noqa: E402
from dbt.parser.sql import SqlBlockParser  # noqa: E402

dbt_version = DbtVersion()

if dbt_version < "v1.8":
    from dbt.contracts.connection import Connection
else:
    from dbt.adapters.contracts.connection import Connection


@contextmanager
def silence_no_nodes_warning():
    if dbt_version >= "v1.8":
        from dbt.events.types import NoNodesForSelectionCriteria
        from dbt_common.events.functions import WARN_ERROR_OPTIONS

        WARN_ERROR_OPTIONS.silence.append(NoNodesForSelectionCriteria.__name__)
    try:
        yield
    finally:
        if dbt_version >= "v1.8":
            from dbt_common.events.functions import WARN_ERROR_OPTIONS

            WARN_ERROR_OPTIONS.silence.pop()


logger = logging.getLogger("uvicorn")
MIN_DBT_NODE_COMPOSITION = 3


class ArtifactsEventHandler(FileSystemEventHandler):
    def __init__(self, watch_files: List[str], callback: Callable = None):
        super().__init__()
        self.watch_files = watch_files
        self.callback = callback

    def on_modified(self, event):
        if event.is_directory:
            return None

        if event.src_path in self.watch_files:
            if callable(self.callback):
                self.callback(event)

    def on_created(self, event):
        if event.is_directory:
            return None

        if event.src_path in self.watch_files:
            if callable(self.callback):
                self.callback(event)


class EnvironmentEventHandler(FileSystemEventHandler):
    def __init__(self, observer, watch_files: set[str], callback: Callable = None):
        super().__init__()
        self.observer = observer
        self.watch_files = watch_files
        self.detected_files = set()
        self.callback = callback

    def on_created(self, event):
        if event.is_directory:
            return

        if event.src_path in self.watch_files:
            self.detected_files.add(event.src_path)

        # Check if all target-base files are created
        if self.detected_files == self.watch_files:
            if callable(self.callback):
                self.callback()
            self.observer.stop()


def merge_tables(tables: List[agate.Table]) -> agate.Table:
    if dbt_version < "v1.8":
        from dbt.clients.agate_helper import merge_tables

        return merge_tables(tables)
    else:
        from dbt_common.clients.agate_helper import merge_tables

        return merge_tables(tables)


def as_manifest(m: WritableManifest) -> Manifest:
    if dbt_version < "v1.8":
        data = m.__dict__
        all_fields = set([x.name for x in fields(Manifest)])
        new_data = {k: v for k, v in data.items() if k in all_fields}
        return Manifest(**new_data)
    else:
        result = Manifest.from_writable_manifest(m)
        result.metadata = ManifestMetadata(**m.metadata.__dict__)
        return result


@track_timing(record_size=True)
def load_manifest(path: str = None, data: dict = None):
    if path is not None:
        if not os.path.isfile(path):
            return None
        return WritableManifest.read_and_check_versions(path)
    if data is not None:
        return WritableManifest.upgrade_schema_version(data)


@track_timing(record_size=True)
def load_catalog(path: str = None, data: dict = None):
    if path is not None:
        if not os.path.isfile(path):
            return None
        return CatalogArtifact.read_and_check_versions(path)
    if data is not None:
        return CatalogArtifact.upgrade_schema_version(data)


def previous_state(state_path: Path, target_path: Path, project_root: Path) -> PreviousState:
    if dbt_version < "v1.5.2":
        return PreviousState(state_path, target_path)
    else:
        try:
            # Overwrite the level_tag method temporarily to avoid the warning message
            from dbt.events.types import EventLevel, WarnStateTargetEqual

            original_level_tag_func = WarnStateTargetEqual.level_tag
            WarnStateTargetEqual.level_tag = lambda x: EventLevel.DEBUG
        except ImportError:
            # Skip overwriting the level_tag method if the dbt version not support
            original_level_tag_func = None
            pass

        state = PreviousState(state_path, target_path, project_root)

        if original_level_tag_func is not None:
            # Restore the original level_tag method
            WarnStateTargetEqual.level_tag = original_level_tag_func

        return state


def default_profiles_dir():
    # Precedence: DBT_PROFILES_DIR > current working directory > ~/.dbt/
    # https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles#advanced-customizing-a-profile-directory
    if os.getenv("DBT_PROFILES_DIR"):
        return os.getenv("DBT_PROFILES_DIR")
    elif os.path.exists(os.path.join(os.getcwd(), "profiles.yml")):
        return os.getcwd()
    else:
        return os.path.expanduser("~/.dbt/")


@dataclass()
class DbtArgs:
    """
    Used for RuntimeConfig.from_args
    """

    threads: Optional[int] = (1,)
    target: Optional[str] = (None,)
    profiles_dir: Optional[str] = (None,)
    project_dir: Optional[str] = (None,)
    profile: Optional[str] = (None,)
    target_path: Optional[str] = (None,)
    project_only_flags: Optional[Dict[str, Any]] = None
    which: Optional[str] = None
    # Behavior flags - need to be present on args object for set_from_args
    state_modified_compare_more_unrendered_values: Optional[bool] = True  # dbt v1.9
    require_unique_project_resource_names: Optional[bool] = False  # dbt v1.11
    require_ref_searches_node_package_before_root: Optional[bool] = False  # dbt v1.11


@dataclass
class DbtAdapter(BaseAdapter):
    runtime_config: RuntimeConfig = None
    adapter: SQLAdapter = None
    manifest: Manifest = None
    previous_state: PreviousState = None
    target_path: str = None
    curr_manifest: WritableManifest = None
    curr_catalog: CatalogArtifact = None
    base_path: str = None
    base_manifest: WritableManifest = None
    base_catalog: CatalogArtifact = None

    # Review mode
    review_mode: bool = False

    # Watch the artifact change
    artifacts_observer = Observer()
    artifacts_files = []

    # Watch the base environment ready
    base_env_observer = Observer()

    def support_tasks(self):
        support_map = {run_type.value: True for run_type in dbt_supported_registry}

        return support_map

    @classmethod
    def load(cls, no_artifacts=False, review=False, **kwargs):
        target = kwargs.get("target")
        target_path = kwargs.get("target_path", "target")
        target_base_path = kwargs.get("target_base_path", "target-base")

        profile_name = kwargs.get("profile")
        project_dir = kwargs.get("project_dir")
        profiles_dir = kwargs.get("profiles_dir")

        if profiles_dir is None:
            profiles_dir = default_profiles_dir()

        # runtime_config
        args = DbtArgs(
            threads=1,
            target=target,
            target_path=target_path,
            project_dir=project_dir,
            profiles_dir=profiles_dir,
            profile=profile_name,
            project_only_flags={},
            which="list",
        )
        set_from_args(args, args)

        from dbt.exceptions import DbtProjectError

        try:
            # adapter
            if dbt_version < "v1.8":
                runtime_config = RuntimeConfig.from_args(args)
                adapter_name = runtime_config.credentials.type
                adapter_cls = get_adapter_class_by_name(adapter_name)
                adapter: SQLAdapter = adapter_cls(runtime_config)
            else:
                from dbt.mp_context import get_mp_context
                from dbt_common.context import (
                    get_invocation_context,
                    set_invocation_context,
                )

                set_invocation_context({})
                get_invocation_context()._env = dict(os.environ)
                runtime_config = RuntimeConfig.from_args(args)
                adapter_name = runtime_config.credentials.type
                adapter_cls = get_adapter_class_by_name(adapter_name)
                adapter: SQLAdapter = adapter_cls(runtime_config, get_mp_context())
                from dbt.adapters.factory import FACTORY

                FACTORY.adapters[adapter_name] = adapter

            adapter.connections.set_connection_name()
            runtime_config.adapter = adapter

            dbt_adapter = cls(
                runtime_config=runtime_config,
                adapter=adapter,
                review_mode=review,
                base_path=target_base_path,
            )
        except DbtProjectError as e:
            raise e

        # Load the artifacts from the state file or dbt target and dbt base directory
        if not no_artifacts and not review:
            dbt_adapter.load_artifacts()
        return dbt_adapter

    def print_lineage_info(self):
        print("Base:")
        print(f"    Manifest: {self.base_manifest.metadata.generated_at}")
        print(f"    Catalog:  {self.base_catalog.metadata.generated_at if self.base_catalog else 'N/A'}")

        print("Current:")
        print(f"    Manifest: {self.curr_manifest.metadata.generated_at}")
        print(f"    Catalog:  {self.curr_catalog.metadata.generated_at if self.curr_catalog else 'N/A'}")

    def get_columns(self, model: str, base=False) -> List[Column]:
        relation = self.create_relation(model, base)
        get_columns_macro = "get_columns_in_relation"
        if self.adapter.connections.TYPE == "databricks":
            get_columns_macro = "get_columns_comments"

        if dbt_version < "v1.8":
            columns = self.adapter.execute_macro(
                get_columns_macro, kwargs={"relation": relation}, manifest=self.manifest
            )
        else:
            from dbt.context.providers import generate_runtime_macro_context

            macro_manifest = MacroManifest(self.manifest.macros)
            self.adapter.set_macro_resolver(macro_manifest)
            self.adapter.set_macro_context_generator(generate_runtime_macro_context)
            columns = self.adapter.execute_macro(get_columns_macro, kwargs={"relation": relation})

        if self.adapter.connections.TYPE == "databricks":
            # reference: get_columns_in_relation (dbt/adapters/databricks/impl.py)
            from dbt.adapters.databricks.column import DatabricksColumn

            rows = columns
            columns = []
            for row in rows:
                if row["col_name"].startswith("#"):
                    break
                columns.append(
                    DatabricksColumn(
                        column=row["col_name"],
                        dtype=row["data_type"],
                        comment=row["comment"],
                    )
                )
            return columns
        else:
            return columns

    def get_model(self, model_id: str, base=False):
        manifest = self.curr_manifest if base is False else self.base_manifest
        manifest_dict = manifest.to_dict()

        node = manifest_dict["nodes"].get(model_id)
        if node is None:
            return {}

        node_name = node["name"]
        with self.adapter.connection_named("model"):
            columns = [column for column in self.get_columns(node_name, base=base)]

        child_map: List[str] = manifest_dict["child_map"][model_id]
        cols_not_null = []
        cols_unique = []

        for child in child_map:
            comps = child.split(".")
            child_type = comps[0]
            child_name = comps[2]

            not_null_prefix = f"not_null_{node_name}_"
            if child_type == "test" and child_name.startswith(not_null_prefix):
                cols_not_null.append(child_name[len(not_null_prefix) :])
            unique_prefix = f"unique_{node_name}_"
            if child_type == "test" and child_name.startswith(unique_prefix):
                cols_unique.append(child_name[len(unique_prefix) :])

        columns_info = {}
        primary_key = None
        for c in columns:
            col_name = c.column
            col = dict(name=col_name, type=c.dtype)
            if col_name in cols_not_null:
                col["not_null"] = True
            if col_name in cols_unique:
                col["unique"] = True
                if not primary_key:
                    primary_key = col_name
            columns_info[col_name] = col

        result = dict(columns=columns_info)
        if primary_key:
            result["primary_key"] = primary_key

        return result

    @track_timing("artifact_load")
    def load_artifacts(self):
        """
        Load the artifacts from the 'target' and 'target-base' directory
        """
        if self.runtime_config is None:
            raise Exception("Cannot find the dbt project configuration")

        project_root = self.runtime_config.project_root
        target_path = self.runtime_config.target_path
        target_base_path = self.base_path
        self.target_path = os.path.join(project_root, target_path)
        self.base_path = os.path.join(project_root, target_base_path)

        # load the artifacts
        path = os.path.join(project_root, target_path, "manifest.json")
        curr_manifest = load_manifest(path=path, timing_name="curr_manifest")
        if curr_manifest is None:
            raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)
        path = os.path.join(project_root, target_base_path, "manifest.json")
        base_manifest = load_manifest(path=path, timing_name="base_manifest")
        if base_manifest is None:
            raise FileNotFoundError(ENOENT, os.strerror(ENOENT), path)

        curr_catalog = load_catalog(
            path=os.path.join(project_root, target_path, "catalog.json"), timing_name="curr_catalog"
        )
        base_catalog = load_catalog(
            path=os.path.join(project_root, target_base_path, "catalog.json"), timing_name="base_catalog"
        )

        # set the value if all the artifacts are loaded successfully
        self.curr_manifest = curr_manifest
        self.curr_catalog = curr_catalog
        self.base_manifest = base_manifest
        self.base_catalog = base_catalog

        # set the manifest
        self.manifest = as_manifest(curr_manifest)
        self.previous_state = previous_state(
            Path(target_base_path),
            Path(self.runtime_config.target_path),
            Path(self.runtime_config.project_root),
        )

        # set the file paths to watch
        self.artifacts_files = [
            os.path.join(project_root, target_path, "manifest.json"),
            os.path.join(project_root, target_path, "catalog.json"),
            os.path.join(project_root, target_base_path, "manifest.json"),
            os.path.join(project_root, target_base_path, "catalog.json"),
        ]

    def is_python_model(self, node_id: str, base: Optional[bool] = False):
        manifest = self.curr_manifest if base is False else self.base_manifest
        model = manifest.nodes.get(node_id)
        if hasattr(model, "language"):
            return model.language == "python"

        return False

    def find_node_by_name(self, node_name, base=False) -> Optional[ManifestNode]:
        manifest = self.curr_manifest if base is False else self.base_manifest

        for key, node in manifest.nodes.items():
            if node.name == node_name:
                return node

        return None

    def get_node_name_by_id(self, unique_id):
        if unique_id.startswith("source."):
            if unique_id in self.curr_manifest.sources:
                return self.curr_manifest.sources[unique_id].name
            elif unique_id in self.base_manifest.sources:
                return self.base_manifest.sources[unique_id].name
        elif unique_id.startswith("metric."):
            if unique_id in self.curr_manifest.metrics:
                return self.curr_manifest.metrics[unique_id].name
            elif unique_id in self.base_manifest.metrics:
                return self.base_manifest.metrics[unique_id].name
        elif unique_id.startswith("exposure."):
            if unique_id in self.curr_manifest.exposures:
                return self.curr_manifest.exposures[unique_id].name
            elif unique_id in self.base_manifest.exposures:
                return self.base_manifest.exposures[unique_id].name
        elif unique_id.startswith("semantic_model."):
            if unique_id in self.curr_manifest.semantic_models:
                return self.curr_manifest.semantic_models[unique_id].name
            elif unique_id in self.base_manifest.semantic_models:
                return self.base_manifest.semantic_models[unique_id].name
        else:
            if unique_id in self.curr_manifest.nodes:
                return self.curr_manifest.nodes[unique_id].name
            elif unique_id in self.base_manifest.nodes:
                return self.base_manifest.nodes[unique_id].name
            return None

    def get_manifest(self, base: bool):
        return self.curr_manifest if base is False else self.base_manifest

    def generate_sql(
        self,
        sql_template: str,
        base: bool = False,
        context=None,
        provided_manifest=None,
    ):
        if context is None:
            context = {}
        manifest = provided_manifest if provided_manifest is not None else as_manifest(self.get_manifest(base))
        parser = SqlBlockParser(self.runtime_config, manifest, self.runtime_config)

        if dbt_version >= dbt_version.parse("v1.8"):
            from dbt_common.context import (
                get_invocation_context,
                set_invocation_context,
            )

            set_invocation_context({})
            get_invocation_context()._env = dict(os.environ)

        node_id = str("generated_" + uuid.uuid4().hex)
        node = parser.parse_remote(sql_template, node_id)
        process_node(self.runtime_config, manifest, node)

        if dbt_version < dbt_version.parse("v1.8"):
            compiler = self.adapter.get_compiler()
            compiler.compile_node(node, manifest, context)
            return node.compiled_code
        else:
            from dbt.clients import jinja
            from dbt.context.providers import (
                generate_runtime_macro_context,
                generate_runtime_model_context,
            )

            # Set up macro resolver for dbt >= 1.8
            macro_manifest = MacroManifest(manifest.macros)
            self.adapter.set_macro_resolver(macro_manifest)
            self.adapter.set_macro_context_generator(generate_runtime_macro_context)

            jinja_ctx = generate_runtime_model_context(node, self.runtime_config, manifest)
            jinja_ctx.update(context)
            compiled_code = jinja.get_rendered(sql_template, jinja_ctx, node)
            return compiled_code

    def execute(
        self,
        sql: str,
        auto_begin: bool = False,
        fetch: bool = False,
        limit: Optional[int] = None,
    ) -> Tuple[any, agate.Table]:
        if dbt_version < dbt_version.parse("v1.6"):
            return self.adapter.execute(sql, auto_begin=auto_begin, fetch=fetch)

        return self.adapter.execute(sql, auto_begin=auto_begin, fetch=fetch, limit=limit)

    def build_parent_map(self, nodes: Dict, base: Optional[bool] = False) -> Dict[str, List[str]]:
        manifest = self.curr_manifest if base is False else self.base_manifest
        manifest_dict = manifest.to_dict()

        node_ids = nodes.keys()
        parent_map = {}
        for k, parents in manifest_dict["parent_map"].items():
            if k not in node_ids:
                continue
            parent_map[k] = [parent for parent in parents if parent in node_ids]

        return parent_map

    def build_parent_list_per_node(self, node_id: str, base: Optional[bool] = False) -> List[str]:
        manifest = self.curr_manifest if base is False else self.base_manifest
        manifest_dict = manifest.to_dict()

        if node_id in manifest_dict["parent_map"]:
            return manifest_dict["parent_map"][node_id]

    def get_lineage(self, base: Optional[bool] = False):
        manifest = self.curr_manifest if base is False else self.base_manifest
        catalog = self.curr_catalog if base is False else self.base_catalog
        cache_key = hash((id(manifest), id(catalog)))
        return self.get_lineage_cached(base, cache_key)

    def get_lineage_diff(self) -> LineageDiff:
        cache_key = hash(
            (
                id(self.base_manifest),
                id(self.base_catalog),
                id(self.curr_manifest),
                id(self.curr_catalog),
            )
        )
        return self._get_lineage_diff_cached(cache_key)

    @lru_cache(maxsize=2)
    def get_lineage_cached(self, base: Optional[bool] = False, cache_key=0):
        if base is False:
            perf_tracker = LineagePerfTracker()
            perf_tracker.start_lineage()

        manifest = self.curr_manifest if base is False else self.base_manifest
        catalog = self.curr_catalog if base is False else self.base_catalog

        manifest_metadata = manifest.metadata if manifest is not None else None
        catalog_metadata = catalog.metadata if catalog is not None else None

        manifest_dict = manifest.to_dict()

        nodes = {}

        for node in manifest_dict["nodes"].values():
            unique_id = node["unique_id"]
            resource_type = node["resource_type"]

            if resource_type not in ["model", "seed", "exposure", "snapshot"]:
                continue

            nodes[unique_id] = {
                "id": node["unique_id"],
                "name": node["name"],
                "resource_type": node["resource_type"],
                "package_name": node["package_name"],
                "schema": node["schema"],
                "config": node["config"],
                "checksum": node["checksum"],
                "raw_code": node["raw_code"],
            }

            # List of <type>.<package_name>.<node_name>.<hash>
            # model.jaffle_shop.customer_segments
            # test.jaffle_shop.not_null_customers_customer_id.5c9bf9911d
            # test.jaffle_shop.unique_customers_customer_id.c5af1ff4b1
            child_map: List[str] = manifest_dict["child_map"][unique_id]
            cols_not_null = []
            cols_unique = []

            for child in child_map:
                node_name = node["name"]
                comps = child.split(".")
                if len(comps) < MIN_DBT_NODE_COMPOSITION:
                    # only happens in unittest
                    continue

                child_type = comps[0]
                child_name = comps[2]

                not_null_prefix = f"not_null_{node_name}_"
                if child_type == "test" and child_name.startswith(not_null_prefix):
                    cols_not_null.append(child_name[len(not_null_prefix) :])
                unique_prefix = f"unique_{node_name}_"
                if child_type == "test" and child_name.startswith(unique_prefix):
                    cols_unique.append(child_name[len(unique_prefix) :])

            if catalog is not None and unique_id in catalog.nodes:
                columns = {}
                primary_key = None
                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
                    col = dict(name=col_name, type=col_metadata.type)
                    if col_name in cols_not_null:
                        col["not_null"] = True
                    if col_name in cols_unique:
                        col["unique"] = True
                        if not primary_key:
                            primary_key = col_name
                    columns[col_name] = col
                nodes[unique_id]["columns"] = columns
                if primary_key:
                    nodes[unique_id]["primary_key"] = primary_key

        for source in manifest_dict["sources"].values():
            unique_id = source["unique_id"]

            nodes[unique_id] = {
                "id": source["unique_id"],
                "name": source["name"],
                "source_name": source["source_name"],
                "resource_type": source["resource_type"],
                "package_name": source["package_name"],
                "config": source["config"],
            }

            if catalog is not None and unique_id in catalog.sources:
                nodes[unique_id]["columns"] = {
                    col_name: {"name": col_name, "type": col_metadata.type}
                    for col_name, col_metadata in catalog.sources[unique_id].columns.items()
                }

        for exposure in manifest_dict["exposures"].values():
            nodes[exposure["unique_id"]] = {
                "id": exposure["unique_id"],
                "name": exposure["name"],
                "resource_type": exposure["resource_type"],
                "package_name": exposure["package_name"],
                "config": exposure["config"],
            }
        for metric in manifest_dict["metrics"].values():
            nodes[metric["unique_id"]] = {
                "id": metric["unique_id"],
                "name": metric["name"],
                "resource_type": metric["resource_type"],
                "package_name": metric["package_name"],
                "config": metric["config"],
            }

        if "semantic_models" in manifest_dict:
            for semantic_models in manifest_dict["semantic_models"].values():
                nodes[semantic_models["unique_id"]] = {
                    "id": semantic_models["unique_id"],
                    "name": semantic_models["name"],
                    "resource_type": semantic_models["resource_type"],
                    "package_name": semantic_models["package_name"],
                    "config": semantic_models["config"],
                }

        parent_map = self.build_parent_map(nodes, base)

        if base is False:
            perf_tracker.end_lineage()
            perf_tracker.set_total_nodes(len(nodes))
            log_performance("model lineage", perf_tracker.to_dict())
            perf_tracker.reset()

        return dict(
            parent_map=parent_map,
            nodes=nodes,
            manifest_metadata=manifest_metadata,
            catalog_metadata=catalog_metadata,
        )

    @lru_cache(maxsize=1)
    def _get_lineage_diff_cached(self, cache_key) -> LineageDiff:
        start_time = time.perf_counter_ns()

        base = self.get_lineage(base=True)
        current = self.get_lineage(base=False)

        select_start = time.perf_counter_ns()
        modified_nodes = self.select_nodes(select="state:modified")
        select_elapsed_ms = (time.perf_counter_ns() - select_start) / 1_000_000

        diff = {}
        for node_id in modified_nodes:
            base_node = base.get("nodes", {}).get(node_id)
            curr_node = current.get("nodes", {}).get(node_id)
            if base_node and curr_node:
                diff[node_id] = NodeDiff(change_status="modified")
            elif base_node:
                diff[node_id] = NodeDiff(change_status="removed")
            elif curr_node:
                diff[node_id] = NodeDiff(change_status="added")

        total_elapsed_ms = (time.perf_counter_ns() - start_time) / 1_000_000
        log_performance(
            "lineage diff",
            {
                "total_elapsed_ms": total_elapsed_ms,
                "select_modified_elapsed_ms": select_elapsed_ms,
                "modified_nodes": len(modified_nodes),
            },
        )

        return LineageDiff(
            base=base,
            current=current,
            diff=diff,
        )

    @lru_cache(maxsize=128)
    def get_change_analysis_cached(self, node_id: str):
        breaking_perf_tracker = BreakingPerformanceTracking()
        lineage_diff = self.get_lineage_diff()
        diff = lineage_diff.diff

        if node_id not in diff or diff[node_id].change_status != "modified":
            return diff.get(node_id)

        breaking_perf_tracker.increment_modified_nodes()
        breaking_perf_tracker.start_lineage_diff()

        base = lineage_diff.base
        current = lineage_diff.current

        base_manifest = as_manifest(self.get_manifest(True))
        curr_manifest = as_manifest(self.get_manifest(False))
        breaking_perf_tracker.record_checkpoint("manifest")

        def ref_func(*args):
            if len(args) == 1:
                node = args[0]
            elif len(args) > 1:
                node = args[1]
            else:
                return None
            return node

        def source_func(source_name, table_name):
            source_name = source_name.replace("-", "_")
            return f"__{source_name}__{table_name}"

        jinja_context = dict(
            ref=ref_func,
            source=source_func,
        )

        base_node = base.get("nodes", {}).get(node_id)
        curr_node = current.get("nodes", {}).get(node_id)
        change = NodeChange(category="unknown")
        if (
            curr_node.get("resource_type") in ["model", "snapshot"]
            and curr_node.get("raw_code") is not None
            and base_node.get("raw_code") is not None
        ):
            try:

                def _get_schema(lineage):
                    schema = {}
                    nodes = lineage["nodes"]
                    parent_list = lineage["parent_map"].get(node_id, [])
                    for parent_id in parent_list:
                        parent_node = nodes.get(parent_id)
                        if parent_node is None:
                            continue
                        columns = parent_node.get("columns") or {}
                        name = parent_node.get("name")
                        if parent_node.get("resource_type") == "source":
                            parts = parent_id.split(".")
                            source = parts[2]
                            table = parts[3]
                            source = source.replace("-", "_")
                            name = f"__{source}__{table}"
                        schema[name] = {name: column.get("type") for name, column in columns.items()}
                    return schema

                base_sql = self.generate_sql(
                    base_node.get("raw_code"),
                    context=jinja_context,
                    provided_manifest=base_manifest,
                )
                curr_sql = self.generate_sql(
                    curr_node.get("raw_code"),
                    context=jinja_context,
                    provided_manifest=curr_manifest,
                )
                base_schema = _get_schema(base)
                curr_schema = _get_schema(current)
                dialect = self.adapter.connections.TYPE
                if curr_manifest.metadata.adapter_type is not None:
                    dialect = curr_manifest.metadata.adapter_type

                change = parse_change_category(
                    base_sql,
                    curr_sql,
                    old_schema=base_schema,
                    new_schema=curr_schema,
                    dialect=dialect,
                    perf_tracking=breaking_perf_tracker,
                )

                # Make sure that the case of the column names are the same
                changed_columns = {
                    column.lower(): change_status for column, change_status in (change.columns or {}).items()
                }
                changed_columns_names = set(changed_columns)
                changed_columns_final = {}

                base_columns = base_node.get("columns") or {}
                curr_columns = curr_node.get("columns") or {}
                columns_names = set(base_columns) | set(curr_columns)

                for column_name in columns_names:
                    if column_name.lower() in changed_columns_names:
                        changed_columns_final[column_name] = changed_columns[column_name.lower()]

                change.columns = changed_columns_final
            except Exception:
                # TODO: telemetry
                pass

        breaking_perf_tracker.end_lineage_diff()
        log_performance("change analysis per node", breaking_perf_tracker.to_dict())
        breaking_perf_tracker.reset()
        node_diff = diff.get(node_id)
        node_diff.change = change
        return node_diff

    def get_cll(
        self,
        node_id: Optional[str] = None,
        column: Optional[str] = None,
        change_analysis: Optional[bool] = False,
        no_cll: Optional[bool] = False,
        no_upstream: Optional[bool] = False,
        no_downstream: Optional[bool] = False,
        no_filter: Optional[bool] = False,
    ) -> CllData:
        cll_tracker = LineagePerfTracker()
        cll_tracker.set_params(
            has_node=node_id is not None,
            has_column=column is not None,
            change_analysis=change_analysis,
            no_cll=no_cll,
            no_upstream=no_upstream,
            no_downstream=no_downstream,
        )
        cll_tracker.start_column_lineage()

        manifest = self.curr_manifest
        manifest_dict = manifest.to_dict()

        # Find related model nodes
        if node_id is not None:
            cll_node_ids = {node_id}
        else:
            lineage_diff = self.get_lineage_diff()
            cll_node_ids = set(lineage_diff.diff.keys())

        cll_tracker.set_init_nodes(len(cll_node_ids))

        nodes = {}
        columns = {}
        parent_map = {}
        child_map = {}

        if not no_upstream:
            cll_node_ids = cll_node_ids.union(find_upstream(cll_node_ids, manifest_dict.get("parent_map")))
        if not no_downstream:
            cll_node_ids = cll_node_ids.union(find_downstream(cll_node_ids, manifest_dict.get("child_map")))

        if not no_cll:
            allowed_related_nodes = set()
            for key in ["sources", "nodes", "exposures", "metrics"]:
                attr = getattr(manifest, key)
                allowed_related_nodes.update(set(attr.keys()))
            if hasattr(manifest, "semantic_models"):
                attr = getattr(manifest, "semantic_models")
                allowed_related_nodes.update(set(attr.keys()))
            for cll_node_id in cll_node_ids:
                if cll_node_id not in allowed_related_nodes:
                    continue
                cll_data_one = deepcopy(self.get_cll_cached(cll_node_id, base=False))
                cll_tracker.increment_cll_nodes()
                if cll_data_one is None:
                    continue

                nodes[cll_node_id] = cll_data_one.nodes.get(cll_node_id)
                node_diff = None
                if change_analysis:
                    node_diff = self.get_change_analysis_cached(cll_node_id)
                    cll_tracker.increment_change_analysis_nodes()
                if node_diff is not None:
                    nodes[cll_node_id].change_status = node_diff.change_status
                    if node_diff.change is not None:
                        nodes[cll_node_id].change_category = node_diff.change.category
                for c_id, c in cll_data_one.columns.items():
                    columns[c_id] = c
                    if node_diff is not None:
                        if node_diff.change_status == "added":
                            c.change_status = "added"
                        elif node_diff.change_status == "removed":
                            c.change_status = "removed"
                        elif node_diff.change is not None and node_diff.change.columns is not None:
                            column_diff = node_diff.change.columns.get(c.name)
                            if column_diff:
                                c.change_status = column_diff

                for p_id, parents in cll_data_one.parent_map.items():
                    parent_map[p_id] = parents
        else:
            for cll_node_id in cll_node_ids:
                cll_node = None
                cll_node_columns: Dict[str, CllColumn] = {}

                if cll_node_id in manifest.sources:
                    cll_node = CllNode.build_cll_node(manifest, "sources", cll_node_id)
                    if self.curr_catalog and cll_node_id in self.curr_catalog.sources:
                        cll_node_columns = {
                            column.name: CllColumn(
                                id=f"{cll_node_id}_{column.name}",
                                table_id=cll_node_id,
                                name=column.name,
                                type=column.type,
                            )
                            for column in self.curr_catalog.sources[cll_node_id].columns.values()
                        }
                elif cll_node_id in manifest.nodes:
                    cll_node = CllNode.build_cll_node(manifest, "nodes", cll_node_id)
                    if self.curr_catalog and cll_node_id in self.curr_catalog.nodes:
                        cll_node_columns = {
                            column.name: CllColumn(
                                id=f"{cll_node_id}_{column.name}",
                                table_id=cll_node_id,
                                name=column.name,
                                type=column.type,
                            )
                            for column in self.curr_catalog.nodes[cll_node_id].columns.values()
                        }
                elif cll_node_id in manifest.exposures:
                    cll_node = CllNode.build_cll_node(manifest, "exposures", cll_node_id)
                elif hasattr(manifest, "semantic_models") and cll_node_id in manifest.semantic_models:
                    cll_node = CllNode.build_cll_node(manifest, "semantic_models", cll_node_id)
                elif cll_node_id in manifest.metrics:
                    cll_node = CllNode.build_cll_node(manifest, "metrics", cll_node_id)

                if not cll_node:
                    continue
                nodes[cll_node_id] = cll_node

                node_diff = None
                if change_analysis:
                    node_diff = self.get_change_analysis_cached(cll_node_id)
                    cll_tracker.increment_change_analysis_nodes()
                if node_diff is not None:
                    cll_node.change_status = node_diff.change_status
                    if node_diff.change is not None:
                        cll_node.change_category = node_diff.change.category
                        for c, cll_column in cll_node_columns.items():
                            cll_node.columns[c] = cll_column
                            columns[cll_column.id] = cll_column
                            if node_diff.change.columns and c in node_diff.change.columns:
                                cll_column.change_status = node_diff.change.columns[c]

                parent_map[cll_node_id] = manifest.parent_map.get(cll_node_id, [])

        # build the child map
        for parent_id, parents in parent_map.items():
            for parent in parents:
                if parent not in child_map:
                    child_map[parent] = set()
                child_map[parent].add(parent_id)

        # Find the anchor nodes
        anchor_node_ids = set()
        extra_node_ids = set()
        if node_id is None and column is None:
            if change_analysis:
                # If change analysis is requested, we need to find the nodes that have changes
                lineage_diff = self.get_lineage_diff()
                for nid, nd in lineage_diff.diff.items():
                    if nd.change_status == "added":
                        anchor_node_ids.add(nid)
                        n = lineage_diff.current["nodes"].get(nid)
                        n_columns = n.get("columns", {})
                        for c in n_columns:
                            anchor_node_ids.add(build_column_key(nid, c))
                        continue
                    if nd.change_status == "removed":
                        extra_node_ids.add(nid)
                        continue

                    node_diff = self.get_change_analysis_cached(nid)
                    if node_diff is not None and node_diff.change is not None:
                        extra_node_ids.add(nid)
                        if no_cll:
                            if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
                                anchor_node_ids.add(nid)
                        else:
                            if node_diff.change.category in ["breaking", "unknown"]:
                                anchor_node_ids.add(nid)
                        if node_diff.change.columns is not None:
                            for column_name in node_diff.change.columns:
                                anchor_node_ids.add(f"{nid}_{column_name}")
            else:
                lineage_diff = self.get_lineage_diff()
                anchor_node_ids = lineage_diff.diff.keys()
        elif node_id is not None and column is None:
            if change_analysis:
                # If change analysis is requested, we need to find the nodes that have changes
                node_diff = self.get_change_analysis_cached(node_id)
                if node_diff is not None and node_diff.change is not None:
                    extra_node_ids.add(node_id)
                    if no_cll:
                        if node_diff.change.category in ["breaking", "partial_breaking", "unknown"]:
                            anchor_node_ids.add(node_id)
                    else:
                        if node_diff.change.category in ["breaking", "unknown"]:
                            anchor_node_ids.add(node_id)
                    if node_diff.change.columns is not None:
                        for column_name in node_diff.change.columns:
                            anchor_node_ids.add(f"{node_id}_{column_name}")
                else:
                    anchor_node_ids.add(node_id)
            else:
                anchor_node_ids.add(node_id)
                if not no_cll:
                    node = nodes.get(node_id)
                    if node:
                        for column_name in node.columns:
                            column_key = build_column_key(node_id, column_name)
                            anchor_node_ids.add(column_key)
        else:
            anchor_node_ids.add(f"{node_id}_{column}")

        cll_tracker.set_anchor_nodes(len(anchor_node_ids))
        result_node_ids = set(anchor_node_ids)
        if not no_upstream:
            result_node_ids = result_node_ids.union(find_upstream(anchor_node_ids, parent_map))
        if not no_downstream:
            result_node_ids = result_node_ids.union(find_downstream(anchor_node_ids, child_map))

        # Filter the nodes and columns based on the anchor nodes
        if not no_filter:
            nodes = {k: v for k, v in nodes.items() if k in result_node_ids or k in extra_node_ids}
            columns = {k: v for k, v in columns.items() if k in result_node_ids or k in extra_node_ids}

            for node in nodes.values():
                node.columns = {
                    k: v for k, v in node.columns.items() if v.id in result_node_ids or v.id in extra_node_ids
                }

                if change_analysis:
                    node.impacted = node.id in result_node_ids

            parent_map, child_map = filter_dependency_maps(parent_map, child_map, result_node_ids)

        cll_tracker.end_column_lineage()
        cll_tracker.set_total_nodes(len(nodes) + len(columns))
        log_performance("column level lineage", cll_tracker.to_dict())
        cll_tracker.reset()

        return CllData(
            nodes=nodes,
            columns=columns,
            parent_map=parent_map,
            child_map=child_map,
        )

    @lru_cache(maxsize=128)
    def get_cll_cached(self, node_id: str, base: Optional[bool] = False) -> Optional[CllData]:
        cll_tracker = CLLPerformanceTracking()

        node, parent_list = self.get_cll_node(node_id, base=base)
        if node is None:
            return None

        cll_tracker.set_total_nodes(1)
        cll_tracker.start_column_lineage()

        def _apply_all_columns(node: CllNode, transformation_type):
            cll_data = CllData()
            cll_data.nodes[node.id] = node
            cll_data.parent_map[node.id] = set(parent_list)
            for col in node.columns.values():
                column_id = f"{node.id}_{col.name}"
                col.transformation_type = transformation_type
                cll_data.columns[column_id] = col
                cll_data.parent_map[column_id] = set()
            return cll_data

        manifest = as_manifest(self.get_manifest(base))
        catalog = self.curr_catalog if base is False else self.base_catalog
        resource_type = node.resource_type
        if resource_type not in {"model", "seed", "source", "snapshot"}:
            return _apply_all_columns(node, "unknown")

        if resource_type == "source" or resource_type == "seed":
            return _apply_all_columns(node, "source")

        if node.raw_code is None or self.is_python_model(node.id, base=base):
            return _apply_all_columns(node, "unknown")

        if node.name == "metricflow_time_spine":
            return _apply_all_columns(node, "source")

        if not node.columns:
            return _apply_all_columns(node, "unknown")

        table_id_map = {}

        def ref_func(*args):
            node_name: str = None
            project_or_package: str = None

            if len(args) == 1:
                node_name = args[0]
            else:
                project_or_package = args[0]
                node_name = args[1]

            for key, n in manifest.nodes.items():
                if n.name != node_name:
                    continue
                if project_or_package is not None and n.package_name != project_or_package:
                    continue

                # replace id "." to "_"
                unique_id = n.unique_id
                table_name = unique_id.replace(".", "_")
                table_id_map[table_name.lower()] = unique_id
                return table_name

            raise ValueError(f"Cannot find node {node_name} in the manifest")

        def source_func(source_name, name):
            for key, n in manifest.sources.items():
                if n.source_name != source_name:
                    continue
                if n.name != name:
                    continue

                # replace id "." to "_"
                unique_id = n.unique_id
                table_name = unique_id.replace(".", "_")
                table_id_map[table_name.lower()] = unique_id
                return table_name

            raise ValueError(f"Cannot find source {source_name}.{name} in the manifest")

        raw_code = node.raw_code
        jinja_context = dict(
            ref=ref_func,
            source=source_func,
        )

        schema = {}
        if catalog is not None:
            for parent_id in parent_list:
                table_name = parent_id.replace(".", "_")
                columns = {}
                if parent_id in catalog.nodes:
                    for col_name, col_metadata in catalog.nodes[parent_id].columns.items():
                        columns[col_name] = col_metadata.type
                if parent_id in catalog.sources:
                    for col_name, col_metadata in catalog.sources[parent_id].columns.items():
                        columns[col_name] = col_metadata.type
                schema[table_name] = columns

        try:
            compiled_sql = self.generate_sql(raw_code, base=base, context=jinja_context, provided_manifest=manifest)
            dialect = self.adapter.type()
            if self.get_manifest(base).metadata.adapter_type is not None:
                dialect = self.get_manifest(base).metadata.adapter_type
            m2c, c2c_map = cll(compiled_sql, schema=schema, dialect=dialect)
        except RecceException:
            cll_tracker.increment_sqlglot_error_nodes()
            return _apply_all_columns(node, "unknown")
        except Exception:
            cll_tracker.increment_other_error_nodes()
            return _apply_all_columns(node, "unknown")

        # Add cll dependency to the node.
        cll_data = CllData()
        cll_data.nodes[node.id] = node
        cll_data.columns = {f"{node.id}_{col.name}": col for col in node.columns.values()}

        # parent map for node
        depends_on = set(parent_list)
        for d in m2c:
            parent_key = f"{table_id_map[d.node.lower()]}_{d.column}"
            depends_on.add(parent_key)
        cll_data.parent_map[node_id] = depends_on

        # parent map for columns
        for name, column in node.columns.items():
            depends_on = set()
            column_id = f"{node.id}_{name}"
            if name in c2c_map:
                for d in c2c_map[name].depends_on:
                    parent_key = f"{table_id_map[d.node.lower()]}_{d.column}"
                    depends_on.add(parent_key)
                column.transformation_type = c2c_map[name].transformation_type
            cll_data.parent_map[column_id] = set(depends_on)

        cll_tracker.end_column_lineage()
        log_performance("column level lineage per node", cll_tracker.to_dict())
        cll_tracker.reset()
        return cll_data

    def get_cll_node(self, node_id: str, base: Optional[bool] = False) -> Tuple[Optional[CllNode], list[str]]:
        manifest = self.curr_manifest if base is False else self.base_manifest
        catalog = self.curr_catalog if base is False else self.base_catalog
        parent_list = []
        node = None

        # model, seed, snapshot
        if node_id in manifest.nodes:
            found = manifest.nodes[node_id]
            unique_id = found.unique_id
            node = CllNode.build_cll_node(manifest, "nodes", node_id)
            if hasattr(found.depends_on, "nodes"):
                parent_list = found.depends_on.nodes

            if catalog is not None and node is not None and unique_id in catalog.nodes:
                columns = {}
                for col_name, col_metadata in catalog.nodes[unique_id].columns.items():
                    column_id = f"{unique_id}_{col_name}"
                    col = CllColumn(id=column_id, name=col_name, table_id=unique_id, type=col_metadata.type)
                    columns[col_name] = col
                node.columns = columns

        # source
        if node_id in manifest.sources:
            found = manifest.sources[node_id]
            unique_id = found.unique_id
            node = CllNode.build_cll_node(manifest, "sources", node_id)
            parent_list = []

            if catalog is not None and node is not None and unique_id in catalog.sources:
                columns = {}
                for col_name, col_metadata in catalog.sources[unique_id].columns.items():
                    column_id = f"{unique_id}_{col_name}"
                    col = CllColumn(id=column_id, name=col_name, table_id=unique_id, type=col_metadata.type)
                    columns[col_name] = col
                node.columns = columns

        # exposure
        if node_id in manifest.exposures:
            found = manifest.exposures[node_id]
            node = CllNode.build_cll_node(manifest, "exposures", node_id)
            if hasattr(found.depends_on, "nodes"):
                parent_list = found.depends_on.nodes

        if hasattr(manifest, "semantic_models") and node_id in manifest.semantic_models:
            found = manifest.semantic_models[node_id]
            node = CllNode.build_cll_node(manifest, "semantic_models", node_id)
            if hasattr(found.depends_on, "nodes"):
                parent_list = found.depends_on.nodes

        if node_id in manifest.metrics:
            found = manifest.metrics[node_id]
            node = CllNode.build_cll_node(manifest, "metrics", node_id)
            if hasattr(found.depends_on, "nodes"):
                parent_list = found.depends_on.nodes

        return node, parent_list

    def get_manifests_by_id(self, unique_id: str):
        curr_manifest = self.get_manifest(base=False)
        base_manifest = self.get_manifest(base=True)
        if unique_id in curr_manifest.nodes.keys() or unique_id in base_manifest.nodes.keys():
            return {
                "current": curr_manifest.nodes.get(unique_id),
                "base": base_manifest.nodes.get(unique_id),
            }
        return None

    def build_name_to_unique_id_index(self) -> Dict[str, str]:
        name_to_unique_id = {}
        curr_manifest = self.get_manifest(base=False)
        base_manifest = self.get_manifest(base=True)

        for unique_id, node in base_manifest.nodes.items():
            name_to_unique_id[node.name] = unique_id
        for unique_id, node in curr_manifest.nodes.items():
            name_to_unique_id[node.name] = unique_id
        return name_to_unique_id

    def start_monitor_artifacts(self, callback: Callable = None):
        if self.artifacts_files:
            event_handler = ArtifactsEventHandler(self.artifacts_files, callback=callback)
            if self.target_path:
                self.artifacts_observer.schedule(event_handler, self.target_path, recursive=False)
            if self.base_path:
                self.artifacts_observer.schedule(event_handler, self.base_path, recursive=False)
            self.artifacts_observer.start()
            logger.info("Start monitoring dbt artifacts")

    def stop_monitor_artifacts(self):
        if self.artifacts_files:
            self.artifacts_observer.stop()
            self.artifacts_observer.join()
            logger.info("Stop monitoring artifacts")

    def start_monitor_base_env(self, callback: Callable = None):
        target_base_dir = os.path.join(self.runtime_config.project_root, "target-base")
        base_env_files = {
            os.path.join(target_base_dir, "manifest.json"),
            os.path.join(target_base_dir, "catalog.json"),
        }
        event_handler = EnvironmentEventHandler(self.base_env_observer, base_env_files, callback=callback)
        self.base_env_observer.schedule(event_handler, self.runtime_config.project_root, recursive=True)
        self.base_env_observer.start()
        logger.info("Start monitoring base environment")

    def stop_monitor_base_env(self):
        if self.base_env_observer.is_alive():
            self.base_env_observer.stop()
        self.base_env_observer.join()
        logger.info("Stop monitoring base environment")

    def set_artifacts(
        self,
        base_manifest: WritableManifest,
        curr_manifest: WritableManifest,
        manifest: Manifest,
        previous_manifest: Manifest,
        base_catalog: CatalogArtifact,
        curr_catalog: CatalogArtifact,
    ):
        self.curr_manifest = curr_manifest
        self.base_manifest = base_manifest
        self.manifest = manifest
        self.curr_catalog = curr_catalog
        self.base_catalog = base_catalog
        self.previous_state = previous_state(
            Path(self.base_path),
            Path(self.runtime_config.target_path),
            Path(self.runtime_config.project_root),
        )
        self.previous_state.manifest = previous_manifest

        # The dependencies of the review mode is derived from manifests.
        # It is a workaround solution to use macro dispatch
        # see: https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch
        dependencies = {}
        for macro in self.manifest.macros.values():
            if macro.package_name not in dependencies:
                dependencies[macro.package_name] = self.runtime_config
        self.runtime_config.dependencies = dependencies

    def refresh(self, refresh_file_path: str = None):
        # Refresh the artifacts
        if refresh_file_path is None:
            return self.load_artifacts()

        # In single environment mode (target_path is equal to base_path),
        # we capture the original manifest as base and only update the current
        target_type = os.path.basename(os.path.dirname(refresh_file_path))
        if self.target_path and target_type == os.path.basename(self.target_path):
            if refresh_file_path.endswith("manifest.json"):
                self.curr_manifest = load_manifest(path=refresh_file_path)
                self.manifest = as_manifest(self.curr_manifest)
                self.get_cll_cached.cache_clear()
                self.get_change_analysis_cached.cache_clear()
            elif refresh_file_path.endswith("catalog.json"):
                self.curr_catalog = load_catalog(path=refresh_file_path)
                self.get_cll_cached.cache_clear()
                self.get_change_analysis_cached.cache_clear()
        elif self.base_path and target_type == os.path.basename(self.base_path):
            if refresh_file_path.endswith("manifest.json"):
                self.base_manifest = load_manifest(path=refresh_file_path)
                self.get_change_analysis_cached.cache_clear()
            elif refresh_file_path.endswith("catalog.json"):
                self.base_catalog = load_catalog(path=refresh_file_path)
                self.get_change_analysis_cached.cache_clear()

    def create_relation(self, model, base=False):
        node = self.find_node_by_name(model, base)
        if node is None:
            return None

        return self.adapter.Relation.create_from(self.runtime_config, node)

    def select_nodes(
        self,
        select: Optional[str] = None,
        exclude: Optional[str] = None,
        packages: Optional[list[str]] = None,
        view_mode: Optional[Literal["all", "changed_models"]] = None,
    ) -> Set[str]:
        import dbt.compilation
        from dbt.compilation import Compiler
        from dbt.graph import (
            NodeSelector,
            SelectionIntersection,
            SelectionUnion,
            parse_difference,
        )

        select_list = [select] if select else None
        exclude_list = [exclude] if exclude else None

        def _parse_difference(include, exclude):
            if dbt_version < "v1.8":
                return parse_difference(include, exclude, "eager")
            else:
                return parse_difference(include, exclude)

        specs = [_parse_difference(select_list, exclude_list)]

        # If packages is not provided, use the project name from manifest metadata as default
        if packages is None:
            if (
                self.manifest.metadata
                and hasattr(self.manifest.metadata, "project_name")
                and self.manifest.metadata.project_name
            ):
                packages = [self.manifest.metadata.project_name]

        if packages is not None:
            package_spec = SelectionUnion([_parse_difference([f"package:{p}"], None) for p in packages])
            specs.append(package_spec)
        if view_mode and view_mode == "changed_models":
            specs.append(_parse_difference(["1+state:modified+"], None))
        spec = SelectionIntersection(specs)

        manifest = Manifest()
        manifest.metadata.adapter_type = self.adapter.type()
        manifest_prev = self.previous_state.manifest
        manifest_curr = self.manifest

        manifest.nodes = {**manifest_curr.nodes}
        # # mark a node is removed if the node id is no in the curr nodes
        for node_id, node in manifest_prev.nodes.items():
            if node_id not in manifest.nodes:
                node_dict = node.to_dict()
                if "raw_code" in node_dict:
                    node_dict["raw_code"] = "__removed__"
                node_class = type(node)
                removed_node = node_class.from_dict(node_dict)
                manifest.nodes[node_id] = removed_node

        manifest.macros = {**manifest_prev.macros, **manifest_curr.macros}
        manifest.sources = {**manifest_prev.sources, **manifest_curr.sources}
        manifest.exposures = {**manifest_prev.exposures, **manifest_curr.exposures}
        manifest.metrics = {**manifest_prev.metrics, **manifest_curr.metrics}
        if hasattr(manifest_prev, "semantic_models"):
            manifest.semantic_models = {
                **manifest_prev.semantic_models,
                **manifest_curr.semantic_models,
            }

        compiler = Compiler(self.runtime_config)
        # disable to print compile states
        tmp_func = dbt.compilation.print_compile_stats
        dbt.compilation.print_compile_stats = lambda x: None
        graph = compiler.compile(manifest, write=False)
        dbt.compilation.print_compile_stats = tmp_func
        selector = NodeSelector(graph, manifest, previous_state=self.previous_state)

        # disable "The selection criterion does not match"
        with silence_no_nodes_warning():
            return selector.get_selected(spec)

    def export_artifacts(self) -> ArtifactsRoot:
        """
        Export the artifacts from the current state
        """
        artifacts = ArtifactsRoot()

        def _load_artifact(artifact):
            return artifact.to_dict() if artifact else None

        artifacts.base = {
            "manifest": _load_artifact(self.base_manifest),
            "catalog": _load_artifact(self.base_catalog),
        }
        artifacts.current = {
            "manifest": _load_artifact(self.curr_manifest),
            "catalog": _load_artifact(self.curr_catalog),
        }
        return artifacts

    def export_artifacts_from_file(self) -> ArtifactsRoot:
        """
        Export the artifacts from the state file. This is the old implementation
        """
        artifacts = ArtifactsRoot()
        target_path = self.runtime_config.target_path
        target_base_path = self.base_path

        def _load_artifact(path):
            if not os.path.isfile(path):
                return None

            with open(path, "r", encoding="utf-8") as f:
                json_content = f.read()
                return json.loads(json_content)

        project_root = self.runtime_config.project_root
        artifacts.base = {
            "manifest": _load_artifact(os.path.join(project_root, target_base_path, "manifest.json")),
            "catalog": _load_artifact(os.path.join(project_root, target_base_path, "catalog.json")),
        }
        artifacts.current = {
            "manifest": _load_artifact(os.path.join(project_root, target_path, "manifest.json")),
            "catalog": _load_artifact(os.path.join(project_root, target_path, "catalog.json")),
        }
        return artifacts

    def import_artifacts(self, artifacts: ArtifactsRoot, merge=True):
        # Merge the artifacts from the state file or cloud
        def _select_artifact(
            original: Union[WritableManifest, CatalogArtifact],
            new: Union[WritableManifest, CatalogArtifact],
        ):
            if merge:
                if not original:
                    return new
                if not new:
                    return original
                return original if original.metadata.generated_at > new.metadata.generated_at else new
            else:
                return new

        self.base_manifest = _select_artifact(self.base_manifest, load_manifest(data=artifacts.base.get("manifest")))
        self.curr_manifest = _select_artifact(self.curr_manifest, load_manifest(data=artifacts.current.get("manifest")))
        self.base_catalog = _select_artifact(self.base_catalog, load_catalog(data=artifacts.base.get("catalog")))
        self.curr_catalog = _select_artifact(self.curr_catalog, load_catalog(data=artifacts.current.get("catalog")))

        self.manifest = as_manifest(self.curr_manifest)
        self.previous_state = previous_state(
            Path(self.base_path),
            Path(self.runtime_config.target_path),
            Path(self.runtime_config.project_root),
        )
        self.previous_state.manifest = as_manifest(self.base_manifest)

        # The dependencies of the review mode is derived from manifests.
        # It is a workaround solution to use macro dispatch
        # see: https://docs.getdbt.com/reference/dbt-jinja-functions/dispatch
        dependencies = {}
        for macro in self.manifest.macros.values():
            if macro.package_name not in dependencies:
                dependencies[macro.package_name] = self.runtime_config

        self.runtime_config.dependencies = dependencies

        if not self.curr_manifest or not self.base_manifest:
            raise Exception(
                "No enough dbt artifacts in the state file. Please use the latest recce to generate the recce state"
            )

    @contextmanager
    def connection_named(self, name: str) -> Iterator[None]:
        with self.adapter.connection_named(name):
            yield

    def get_thread_connection(self) -> Connection:
        return self.adapter.connections.get_thread_connection()

    def cancel(self, connection: Connection):
        self.adapter.connections.cancel(connection)

    def __hash__(self):
        return id(self)

    def __eq__(self, other):
        return self.__hash__() == other.__hash__()
