import asyncio
import dataclasses
import itertools
import json
import logging
import os
import re
import sys
import time
import webbrowser
from collections.abc import Mapping, Callable
from pathlib import Path
from typing import NoReturn, TextIO

from databricks.sdk.service.sql import CreateWarehouseRequestWarehouseType
from databricks.sdk import WorkspaceClient

from databricks.labs.blueprint.cli import App
from databricks.labs.blueprint.entrypoint import get_logger
from databricks.labs.blueprint.installation import RootJsonValue, JsonObject, JsonValue
from databricks.labs.blueprint.tui import Prompts


from databricks.labs.lakebridge.assessments.configure_assessment import create_assessment_configurator
from databricks.labs.lakebridge.assessments import PROFILER_SOURCE_SYSTEM, PRODUCT_NAME
from databricks.labs.lakebridge.assessments.profiler import Profiler

from databricks.labs.lakebridge.config import TranspileConfig, LSPConfigOptionV1
from databricks.labs.lakebridge.contexts.application import ApplicationContext
from databricks.labs.lakebridge.connections.credential_manager import cred_file, create_credential_manager
from databricks.labs.lakebridge.connections.database_manager import DatabaseManager
from databricks.labs.lakebridge.connections.env_getter import EnvGetter
from databricks.labs.lakebridge.connections.synapse_connection_helpers import validate_synapse_pools
from databricks.labs.lakebridge.helpers.recon_config_utils import ReconConfigPrompts
from databricks.labs.lakebridge.helpers.telemetry_utils import make_alphanum_or_semver
from databricks.labs.lakebridge.reconcile.runner import ReconcileRunner
from databricks.labs.lakebridge.lineage import lineage_generator
from databricks.labs.lakebridge.reconcile.recon_config import RECONCILE_OPERATION_NAME, AGG_RECONCILE_OPERATION_NAME
from databricks.labs.lakebridge.transpiler.describe import TranspilersDescription
from databricks.labs.lakebridge.transpiler.execute import transpile as do_transpile
from databricks.labs.lakebridge.transpiler.lsp.lsp_engine import LSPEngine
from databricks.labs.lakebridge.transpiler.repository import TranspilerRepository
from databricks.labs.lakebridge.transpiler.sqlglot.sqlglot_engine import SqlglotEngine
from databricks.labs.lakebridge.transpiler.switch_runner import SwitchRunner
from databricks.labs.lakebridge.transpiler.transpile_engine import TranspileEngine

from databricks.labs.lakebridge.transpiler.transpile_status import ErrorSeverity
from databricks.labs.switch.lsp import get_switch_dialects


# Subclass to allow controlled access to protected methods.
class Lakebridge(App):

    def create_workspace_client(self) -> WorkspaceClient:
        """Create a workspace client, with the appropriate product and version information.

        This is intended only for use by the install/uninstall hooks.
        """
        self._patch_databricks_host()
        return self._workspace_client()

    def _log_level(self, raw: str) -> int:
        """Convert the log-level provided by the Databricks CLI into a logging level supported by Python."""
        log_level = super()._log_level(raw)
        # Due to an issue in the handoff of the intended logging level from the Databricks CLI to our
        # application, we can't currently distinguish between --log-level=WARN and nothing at all, where we
        # prefer (and the application logging expects) INFO.
        #
        # Rather than default to only have WARNING logs show, it's preferable to default to INFO and have
        # --log-level=WARN not work for now.
        #
        # See: https://github.com/databrickslabs/lakebridge/issues/2167
        # TODO: Remove this once #2167 has been resolved.
        if log_level == logging.WARNING:
            log_level = logging.INFO
        return log_level


lakebridge = Lakebridge(__file__)
logger = get_logger(__file__)


def raise_validation_exception(msg: str) -> NoReturn:
    raise ValueError(msg)


def _create_warehouse(ws: WorkspaceClient) -> str:

    dbsql = ws.warehouses.create_and_wait(
        name=f"lakebridge-warehouse-{time.time_ns()}",
        warehouse_type=CreateWarehouseRequestWarehouseType.PRO,
        cluster_size="Small",  # Adjust size as needed
        auto_stop_mins=30,  # Auto-stop after 30 minutes of inactivity
        enable_serverless_compute=True,
        max_num_clusters=1,
    )

    if dbsql.id is None:
        raise RuntimeError(f"Failed to create warehouse {dbsql.name}")

    logger.info(f"Created warehouse with id: {dbsql.id}")
    return dbsql.id


def _remove_warehouse(ws: WorkspaceClient, warehouse_id: str):
    ws.warehouses.delete(warehouse_id)
    logger.info(f"Removed warehouse post installation with id: {warehouse_id}")


@lakebridge.command
def transpile(  # pylint: disable=too-many-arguments
    *,
    w: WorkspaceClient,
    transpiler_config_path: str | None = None,
    source_dialect: str | None = None,
    overrides_file: str | None = None,
    target_technology: str | None = None,
    input_source: str | None = None,
    output_folder: str | None = None,
    error_file_path: str | None = None,
    skip_validation: str | None = None,
    catalog_name: str | None = None,
    schema_name: str | None = None,
    ctx: ApplicationContext | None = None,
    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
):
    """Transpiles source dialect to databricks dialect"""
    if ctx is None:
        ctx = ApplicationContext(w)
    del w
    logger.debug(f"Preconfigured transpiler config: {ctx.transpile_config!r}")
    ctx.add_user_agent_extra("cmd", "execute-transpile")
    checker = _TranspileConfigChecker(ctx.transpile_config, ctx.prompts, transpiler_repository)
    checker.use_transpiler_config_path(transpiler_config_path)
    checker.use_source_dialect(source_dialect)
    checker.use_overrides_file(overrides_file)
    checker.use_target_technology(target_technology)
    checker.use_input_source(input_source)
    checker.use_output_folder(output_folder)
    checker.use_error_file_path(error_file_path)
    checker.use_skip_validation(skip_validation)
    checker.use_catalog_name(catalog_name)
    checker.use_schema_name(schema_name)
    config, engine = checker.check()
    logger.debug(f"Final configuration for transpilation: {config!r}")
    _add_user_agent_extras_transpile(ctx, config, engine, transpiler_repository)
    result = asyncio.run(_transpile(ctx, config, engine))
    # DO NOT Modify this print statement, it is used by the CLI to display results in GO Table Template
    print(json.dumps(result))


def _add_user_agent_extras_transpile(
    ctx: ApplicationContext,
    config: TranspileConfig,
    engine: TranspileEngine,
    transpiler_repository: TranspilerRepository,
) -> None:
    assert config.source_dialect is not None, "Source dialect has been validated by this point."
    ctx.add_user_agent_extra("transpiler_source_tech", make_alphanum_or_semver(config.source_dialect))

    plugin_name = engine.transpiler_name
    plugin_name = re.sub(r"\s+", "_", plugin_name)
    ctx.add_user_agent_extra("transpiler_plugin_name", plugin_name)

    config_path = config.transpiler_config_path_parsed
    assert config_path is not None, "Transpiler config path has been validated by this point."
    transpiler_version = transpiler_repository.get_installed_version_given_config_path(config_path)
    if transpiler_version:
        ctx.add_user_agent_extra("transpiler_plugin_version", transpiler_version)
    else:
        logger.warning("Cannot determine transpiler plugin version.")

    # Send telemetry
    user = ctx.current_user
    logger.debug(f"User: {user}")


class _TranspileConfigChecker:
    """Helper class for the 'transpile' command to check and consolidate the configuration."""

    #
    # Configuration parameters can come from 3 sources:
    #  - Command-line arguments (e.g., --input-source, --output-folder, etc.)
    #  - The configuration file, stored in the user's workspace home directory.
    #  - User prompts.
    #
    # The conventions are:
    #  - Command-line arguments take precedence over the configuration file.
    #  - Prompting is a last resort, only used when a required configuration value has not been provided and does not
    #    have a default value.
    #  - An invalid value results in a halt, with the error message indicating the source of the invalid value. We do
    #    NOT attempt to recover from invalid values by looking for another source:
    #     - Prompting unexpectedly will break scripting and automation.
    #     - Using an alternate value will need to confusion because the behaviour will not be what the user expects.
    #
    # This ensures that we distinguish between:
    #  - Invalid command-line arguments:
    #    Resolution: fix the command-line argument value.
    #  - Invalid prompt responses:
    #    Resolution: provide a valid response to the prompt.
    #  - Invalid configuration file values:
    #    Resolution: fix the configuration file value, or provide the command-line argument to override it.
    #
    # Implementation details:
    #  - For command-line arguments and prompted values, we:
    #     - Log the raw values (prior to validation) at DEBUG level, using the repr() rendering.
    #     - Validate the values immediately, with the error message on failure mentioning the source of the value.
    #     - Only update the configuration if the validation passes.
    #  - Prompting only occurs when a value is required, but not provided via the command-line argument or the
    #    configuration file.
    #  - In addition to the above, a final validation of everything is required: this ensures that values from the
    #    configuration file are validated, and if we have a failure we know that's the source because other sources
    #    were already checked.
    #  - The interplay between the source dialect and the transpiler config path is handled with care:
    #      - The source dialect, needs to be consistent with the engine that transpiler config path, refers to.
    #      - The source dialect can be used to infer the transpiler config path.
    #
    # TODO: Refactor this class to eliminate a lof of the boilerplate and handle this more elegantly.

    _config: TranspileConfig
    """The workspace configuration for transpiling, updated from command-line arguments."""
    _prompts: Prompts
    """Prompting system, for requesting configuration that hasn't been provided."""
    _source_dialect_override: str | None = None
    """The source dialect provided on the command-line, if any."""
    _transpiler_repository: TranspilerRepository
    """The repository where available transpilers are installed."""

    def __init__(
        self,
        config: TranspileConfig | None,
        prompts: Prompts,
        transpiler_repository: TranspilerRepository,
    ) -> None:
        if config is None:
            logger.debug("No workspace transpile configuration, starting from defaults.")
            config = TranspileConfig()
        self._config = config
        self._prompts = prompts
        self._transpiler_repository = transpiler_repository
        self._source_dialect_override = None

    @staticmethod
    def _validate_transpiler_config_path(transpiler_config_path: str, msg: str) -> None:
        """Validate the transpiler config path: it must be a valid path that exists."""
        # Note: the content is not validated here, but during loading of the engine.
        if not Path(transpiler_config_path).exists():
            raise_validation_exception(msg)

    def use_transpiler_config_path(self, transpiler_config_path: str | None) -> None:
        if transpiler_config_path is not None:
            logger.debug(f"Setting transpiler_config_path to: {transpiler_config_path!r}")
            self._validate_transpiler_config_path(
                transpiler_config_path,
                f"Invalid path for '--transpiler-config-path', does not exist: {transpiler_config_path}",
            )
            self._config = dataclasses.replace(self._config, transpiler_config_path=transpiler_config_path)

    def use_source_dialect(self, source_dialect: str | None) -> None:
        if source_dialect is not None:
            # Defer validation: depends on the transpiler config path, we'll deal with this later.
            logger.debug(f"Pending source_dialect override: {source_dialect!r}")
            self._source_dialect_override = source_dialect

    @staticmethod
    def _validate_overrides_file(overrides_file: str, msg: str) -> None:
        """Validate the overrides file: it must be a valid path that exists."""
        # Note: in addition to this check, later we verify the transpiler supports it.
        if not Path(overrides_file).exists():
            raise_validation_exception(msg)

    def use_overrides_file(self, overrides_file: str | None) -> None:
        if overrides_file is not None:
            logger.debug(f"Setting overrides_file to: {overrides_file!r}")
            msg = f"Invalid path for '--overrides-file', does not exist: {overrides_file}"
            self._validate_overrides_file(overrides_file, msg)
            try:
                self._set_config_transpiler_option("overrides-file", overrides_file)
            except ValueError:
                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
                msg = "Cannot use --overrides-file; workspace config.yml has incompatible transpiler_options."
                raise_validation_exception(msg)

    def use_target_technology(self, target_technology: str | None) -> None:
        if target_technology is not None:
            logger.debug(f"Setting target_technology to: {target_technology!r}")
            # Cannot validate this here: depends on the transpiler engine, and will be checked later.
            try:
                self._set_config_transpiler_option("target-tech", target_technology)
            except ValueError:
                # TODO: Update the `config.yml` format to disallow incompatible `transpiler_options`.
                msg = "Cannot use --target-technology; workspace config.yml has incompatible transpiler_options."
                raise_validation_exception(msg)

    @staticmethod
    def _validate_input_source(input_source: str, msg: str) -> None:
        """Validate the input source: it must be a path that exists."""
        if not Path(input_source).exists():
            raise_validation_exception(msg)

    def use_input_source(self, input_source: str | None) -> None:
        if input_source is not None:
            logger.debug(f"Setting input_source to: {input_source!r}")
            self._validate_input_source(
                input_source, f"Invalid path for '--input-source', does not exist: {input_source}"
            )
            self._config = dataclasses.replace(self._config, input_source=input_source)

    def _prompt_input_source(self) -> None:
        prompted_input_source = self._prompts.question("Enter input SQL path (directory/file)").strip()
        logger.debug(f"Setting input_source to: {prompted_input_source!r}")
        self._validate_input_source(
            prompted_input_source, f"Invalid input source, path does not exist: {prompted_input_source}"
        )
        self._config = dataclasses.replace(self._config, input_source=prompted_input_source)

    def _check_input_source(self) -> None:
        config_input_source = self._config.input_source
        if config_input_source is None:
            self._prompt_input_source()
        else:
            self._validate_input_source(
                config_input_source, f"Invalid input source path configured, does not exist: {config_input_source}"
            )

    @staticmethod
    def _validate_output_folder(output_folder: str, msg: str) -> None:
        """Validate the output folder: it doesn't have to exist, but its parent must."""
        if not Path(output_folder).parent.exists():
            raise_validation_exception(msg)

    def use_output_folder(self, output_folder: str | None) -> None:
        if output_folder is not None:
            logger.debug(f"Setting output_folder to: {output_folder!r}")
            self._validate_output_folder(
                output_folder, f"Invalid path for '--output-folder', parent does not exist for: {output_folder}"
            )
            self._config = dataclasses.replace(self._config, output_folder=output_folder)

    def _prompt_output_folder(self) -> None:
        prompted_output_folder = self._prompts.question("Enter output folder path (directory)").strip()
        logger.debug(f"Setting output_folder to: {prompted_output_folder!r}")
        self._validate_output_folder(
            prompted_output_folder, f"Invalid output folder path, parent does not exist for: {prompted_output_folder}"
        )
        self._config = dataclasses.replace(self._config, output_folder=prompted_output_folder)

    def _check_output_folder(self) -> None:
        config_output_folder = self._config.output_folder
        if config_output_folder is None:
            self._prompt_output_folder()
        else:
            self._validate_output_folder(
                config_output_folder,
                f"Invalid output folder configured, parent does not exist for: {config_output_folder}",
            )

    @staticmethod
    def _validate_error_file_path(error_file_path: str | None, msg: str) -> None:
        """Value the error file path: it doesn't have to exist, but its parent must."""
        if error_file_path is not None and not Path(error_file_path).parent.exists():
            raise_validation_exception(msg)

    def use_error_file_path(self, error_file_path: str | None) -> None:
        if error_file_path is not None:
            logger.debug(f"Setting error_file_path to: {error_file_path!r}")
            self._validate_error_file_path(
                error_file_path, f"Invalid path for '--error-file-path', parent does not exist: {error_file_path}"
            )
            self._config = dataclasses.replace(self._config, error_file_path=error_file_path)

    def _check_error_file_path(self) -> None:
        config_error_file_path = self._config.error_file_path
        self._validate_error_file_path(
            config_error_file_path,
            f"Invalid error file path configured, parent does not exist for: {config_error_file_path}",
        )

    def use_skip_validation(self, skip_validation: str | None) -> None:
        if skip_validation is not None:
            skip_validation_lower = skip_validation.lower()
            if skip_validation_lower not in {"true", "false"}:
                msg = f"Invalid value for '--skip-validation': {skip_validation!r} must be 'true' or 'false'."
                raise_validation_exception(msg)
            new_skip_validation = skip_validation_lower == "true"
            logger.debug(f"Setting skip_validation to: {new_skip_validation!r}")
            self._config = dataclasses.replace(self._config, skip_validation=new_skip_validation)

    def use_catalog_name(self, catalog_name: str | None) -> None:
        if catalog_name:
            logger.debug(f"Setting catalog_name to: {catalog_name!r}")
            self._config = dataclasses.replace(self._config, catalog_name=catalog_name)

    def use_schema_name(self, schema_name: str | None) -> None:
        if schema_name:
            logger.debug(f"Setting schema_name to: {schema_name!r}")
            self._config = dataclasses.replace(self._config, schema_name=schema_name)

    def _set_config_transpiler_option(self, flag: str, value: str) -> None:
        transpiler_options: JsonObject
        match self._config.transpiler_options:
            case None:
                transpiler_options = {flag: value}
            case Mapping() as found_options:
                transpiler_options = {**found_options, flag: value}
            case found_options:
                # TODO: Update `config.yml' to constrain `transpiler_options` to be a dict[str, str].
                msg = f"Incompatible transpiler options configured, must be a mapping: {found_options!r}"
                raise ValueError(msg)
        self._config = dataclasses.replace(self._config, transpiler_options=transpiler_options)

    def _configure_transpiler_config_path(self, source_dialect: str) -> TranspileEngine | None:
        """Configure the transpiler config path based on the requested source dialect."""
        # Names of compatible transpiler engines for the given dialect.
        compatible_transpilers = self._transpiler_repository.transpilers_with_dialect(source_dialect)
        match len(compatible_transpilers):
            case 0:
                # Nothing found for the specified dialect, fail.
                return None
            case 1:
                # Only one transpiler available for the specified dialect, use it.
                transpiler_name = next(iter(compatible_transpilers))
                logger.debug(f"Using only transpiler available for dialect {source_dialect!r}: {transpiler_name!r}")
            case _:
                # Multiple transpilers available for the specified dialect, prompt for which to use.
                logger.debug(
                    f"Multiple transpilers available for dialect {source_dialect!r}: {compatible_transpilers!r}"
                )
                transpiler_name = self._prompts.choice("Select the transpiler:", list(compatible_transpilers))
        transpiler_config_path = self._transpiler_repository.transpiler_config_path(transpiler_name)
        logger.info(f"Lakebridge will use the {transpiler_name} transpiler.")
        self._config = dataclasses.replace(self._config, transpiler_config_path=str(transpiler_config_path))
        return LSPEngine.from_config_path(transpiler_config_path)

    def _configure_source_dialect(
        self, source_dialect: str, engine: TranspileEngine | None, msg_prefix: str
    ) -> TranspileEngine:
        """Configure the source dialect, if possible, and return the transpiler engine."""
        if engine is None:
            engine = self._configure_transpiler_config_path(source_dialect)
            if engine is None:
                supported_dialects = ", ".join(self._transpiler_repository.all_dialects())
                msg = f"{msg_prefix}: {source_dialect!r} (supported dialects: {supported_dialects})"
                raise_validation_exception(msg)
            else:
                self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
        else:
            # Check the source dialect against the engine.
            if source_dialect not in engine.supported_dialects:
                supported_dialects_description = ", ".join(engine.supported_dialects)
                msg = f"Invalid value for '--source-dialect': {source_dialect!r} must be one of: {supported_dialects_description}"
                raise_validation_exception(msg)
            self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
        return engine

    def _prompt_source_dialect(self) -> TranspileEngine:
        # This is similar to the post-install prompting for the source dialect.
        supported_dialects = self._transpiler_repository.all_dialects()
        match len(supported_dialects):
            case 0:
                msg = "No transpilers are available, install using 'install-transpile' or use --transpiler-conf-path'."
                raise_validation_exception(msg)
            case 1:
                # Only one dialect available, use it.
                source_dialect = next(iter(supported_dialects))
                logger.debug(f"Using only source dialect available: {source_dialect!r}")
            case _:
                # Multiple dialects available, prompt for which to use.
                logger.debug(f"Multiple source dialects available, choice required: {supported_dialects!r}")
                source_dialect = self._prompts.choice("Select the source dialect:", list(supported_dialects))
        engine = self._configure_transpiler_config_path(source_dialect)
        assert engine is not None, "No transpiler engine available for a supported dialect; configuration is invalid."
        self._config = dataclasses.replace(self._config, source_dialect=source_dialect)
        return engine

    def _check_lsp_engine(self) -> TranspileEngine:
        #
        # This is somewhat complicated:
        #  - If there is no transpiler config path, we need to try to infer it from the source dialect.
        #  - If there is no source dialect, we need to prompt for it: but that depends on the transpiler config path.
        #
        # With this in mind, the steps here are:
        # 1. If the transpiler config path is set, check it exists and load the engine.
        # 2. If the source dialect is set,
        #      - If the transpiler config path is set: validate the source dialect against the engine.
        #      - If the transpiler config path is not set: search for a transpiler that satisfies the dialect:
        #          * If one is found, we're good to go.
        #          * If more than one is found, prompt for the transpiler config path.
        #          * If none are found, fail: no transpilers available for the specified dialect.
        #    At this point we have either halted, or we have a valid transpiler path and source dialect.
        # 3. If the source dialect is not set, we need to:
        #      a) Load the set of available dialects: just for the engine if transpiler config path is set, or for all
        #         available transpilers if not.
        #      b) Depending on the available dialects:
        #          - If there is only one dialect available, set it as the source dialect.
        #          - If there are multiple dialects available, prompt for which to use.
        #          - If there are no dialects available, fail: no transpilers available.
        #    At this point we have either halted, or we have a valid transpiler path and source dialect.
        #
        # TODO: Deal with the transpiler options, and filtering them for the engine.
        #

        # Step 1: Check the transpiler config path.
        engine: TranspileEngine | None
        transpiler_config_path = self._config.transpiler_config_path
        if transpiler_config_path is not None:
            self._validate_transpiler_config_path(
                transpiler_config_path,
                f"Error: Invalid value for '--transpiler-config-path': '{str(transpiler_config_path)}', file does not exist.",
            )
            path = Path(transpiler_config_path)
            engine = LSPEngine.from_config_path(path)
        else:
            engine = None
        del transpiler_config_path

        # Step 2: Check the source dialect, assuming it has been specified, and infer the transpiler config path if necessary.
        source_dialect = self._source_dialect_override
        if source_dialect is not None:
            logger.debug(f"Setting source_dialect override: {source_dialect!r}")
            engine = self._configure_source_dialect(source_dialect, engine, "Invalid value for '--source-dialect'")
        else:
            source_dialect = self._config.source_dialect
            if source_dialect is not None:
                logger.debug(f"Using configured source_dialect: {source_dialect!r}")
                engine = self._configure_source_dialect(source_dialect, engine, "Invalid configured source dialect")
            else:
                # Step 3: Source dialect is not set, we need to prompt for it.
                logger.debug("No source_dialect available, prompting.")
                engine = self._prompt_source_dialect()
        return engine

    def _check_transpiler_options(self, engine: TranspileEngine) -> None:
        if not isinstance(engine, LSPEngine):
            return
        assert self._config.source_dialect is not None, "Source dialect must be set before checking transpiler options."
        options_for_dialect = engine.options_for_dialect(self._config.source_dialect)
        transpiler_options = self._config.transpiler_options
        if transpiler_options is None:
            transpiler_options = {}
        elif not isinstance(transpiler_options, Mapping):
            logger.warning(f"Ignoring transpiler_options in config.yml, must be a mapping: {transpiler_options!r}")
            transpiler_options = {}
        # Only checks if the option is present, does not validate the value.
        # TODO: Validate the value for CHOICE/CONFIRM options.
        # TODO: Handle FORCE options: these are fixed by the transpiler, and cannot be overridden.
        checked_options = {
            option.flag: (
                transpiler_options[option.flag]
                if option.flag in transpiler_options
                else self._handle_missing_transpiler_option(option)
            )
            for option in options_for_dialect
        }
        self._config = dataclasses.replace(self._config, transpiler_options=checked_options)

    def _handle_missing_transpiler_option(self, option: LSPConfigOptionV1) -> JsonValue:
        # Semantics during configuration:
        #  - Entries are present in the config file for all options the LSP server needs for a dialect.
        #  - If a value is `None`, it means the user wants the value to be left unset.
        #  - There is no 'provide it later' option: either it's set, or it's unset.
        # As a corner case, if there is no entry present it means the user wasn't prompted. Here we have
        # some complexity. We have two ways of obtaining a value:
        #  - The user could provide it on the command-line, using --target-technology or --overrides-file.
        #    Problem: via command-line options there's no way to indicate 'no value'.
        #  - We could prompt for it, assuming the user is running interactively.
        # In terms of what is required by the option:
        #  - If the option has a default of <none>, it means that no value is required.
        #  - Everything else requires a value.
        #
        # This leads to the following business rules:
        #  - If the option has a default of <none> that means that no value is required, no further action is required.
        #  - Otherwise, a value is required: prompt for it.
        #
        # TODO: When adding non-interactive support, the otherwise branch need to be modified:
        #     1. If it can be provided by the command-line, fail and ask the user to provide it.
        #     2. If it cannot be provided by the command-line, prompt for it if we are running interactively.
        #     3. If we cannot prompt because we are not running interactively, use the default if there is one.
        #     4. Fail: the only way to provide a value is via the config.yml, which can be set via 'install-transpile'.

        if option.is_optional():
            return None
        return option.prompt_for_value(self._prompts)

    def check(self) -> tuple[TranspileConfig, TranspileEngine]:
        """Checks that all configuration parameters are present and valid."""
        logger.debug(f"Checking config: {self._config!r}")

        self._check_input_source()
        self._check_output_folder()
        self._check_error_file_path()
        # No validation here required for:
        #   - skip_validation: it is a boolean flag, mandatory, and has a default: so no further validation is needed.
        #   - catalog_name and schema_name: they are mandatory, but have a default.
        # TODO: if validation is enabled, we should check that the catalog and schema names are valid.

        # This covers: transpiler_config_path, source_dialect
        engine = self._check_lsp_engine()

        # Last thing: the configuration may have transpiler-specific options, check them.
        self._check_transpiler_options(engine)

        config = self._config
        logger.debug(f"Validated config: {config!r}")
        return config, engine


async def _transpile(ctx: ApplicationContext, config: TranspileConfig, engine: TranspileEngine) -> RootJsonValue:
    """Transpiles source dialect to databricks dialect"""
    _override_workspace_client_config(ctx, config.sdk_config)
    status, errors = await do_transpile(ctx.workspace_client, engine, config)

    logger.debug(f"Transpilation completed with status: {status}")

    for path, errors_by_path in itertools.groupby(errors, key=lambda x: x.path):
        errs = list(errors_by_path)
        errors_by_severity = {
            severity.name: len(list(errors)) for severity, errors in itertools.groupby(errs, key=lambda x: x.severity)
        }
        reports = []
        reported_severities = [ErrorSeverity.ERROR, ErrorSeverity.WARNING]
        for severity in reported_severities:
            if severity.name in errors_by_severity:
                word = str.lower(severity.name) + "s" if errors_by_severity[severity.name] > 1 else ""
                reports.append(f"{errors_by_severity[severity.name]} {word}")

        msg = ", ".join(reports) + " found"

        if ErrorSeverity.ERROR.name in errors_by_severity:
            logger.error(f"{path}: {msg}")
        elif ErrorSeverity.WARNING.name in errors_by_severity:
            logger.warning(f"{path}: {msg}")

    # Table Template in labs.yml requires the status to be list of dicts Do not change this
    return [status]


def _override_workspace_client_config(ctx: ApplicationContext, overrides: dict[str, str] | None) -> None:
    """
    Override the Workspace client's SDK config with the user provided SDK config.
    Users can provide the cluster_id and warehouse_id during the installation.
    This will update the default config object in-place.
    """
    if not overrides:
        return

    warehouse_id = overrides.get("warehouse_id")
    if warehouse_id:
        ctx.connect_config.warehouse_id = warehouse_id

    cluster_id = overrides.get("cluster_id")
    if cluster_id:
        ctx.connect_config.cluster_id = cluster_id


@lakebridge.command
def reconcile(
    *, w: WorkspaceClient, ctx_factory: Callable[[WorkspaceClient], ApplicationContext] = ApplicationContext
) -> None:
    """[EXPERIMENTAL] Reconciles source to Databricks datasets"""
    ctx = ctx_factory(w)
    ctx.add_user_agent_extra("cmd", "execute-reconcile")
    user = ctx.current_user
    logger.debug(f"User: {user}")
    recon_runner = ReconcileRunner(
        ctx.workspace_client,
        ctx.install_state,
    )

    _, job_run_url = recon_runner.run(operation_name=RECONCILE_OPERATION_NAME)
    if ctx.prompts.confirm(f"Would you like to open the job run URL `{job_run_url}` in the browser?"):
        webbrowser.open(job_run_url)


@lakebridge.command
def aggregates_reconcile(
    *, w: WorkspaceClient, ctx_factory: Callable[[WorkspaceClient], ApplicationContext] = ApplicationContext
) -> None:
    """[EXPERIMENTAL] Reconciles Aggregated source to Databricks datasets"""
    ctx = ctx_factory(w)
    ctx.add_user_agent_extra("cmd", "execute-aggregates-reconcile")
    user = ctx.current_user
    logger.debug(f"User: {user}")
    recon_runner = ReconcileRunner(
        ctx.workspace_client,
        ctx.install_state,
    )

    _, job_run_url = recon_runner.run(operation_name=AGG_RECONCILE_OPERATION_NAME)
    if ctx.prompts.confirm(f"Would you like to open the job run URL `{job_run_url}` in the browser?"):
        webbrowser.open(job_run_url)


@lakebridge.command
def generate_lineage(
    *,
    w: WorkspaceClient,
    source_dialect: str | None = None,
    input_source: str,
    output_folder: str,
) -> None:
    """[Experimental] Generates a lineage of source SQL files or folder"""
    ctx = ApplicationContext(w)
    logger.debug(f"User: {ctx.current_user}")
    if not os.path.exists(input_source):
        raise_validation_exception(f"Invalid path for '--input-source': Path '{input_source}' does not exist.")
    if not os.path.exists(output_folder):
        raise_validation_exception(f"Invalid path for '--output-folder': Path '{output_folder}' does not exist.")
    if source_dialect is None:
        raise_validation_exception("Value for '--source-dialect' must be provided.")
    engine = SqlglotEngine()
    supported_dialects = engine.supported_dialects
    if source_dialect not in supported_dialects:
        supported_dialects_description = ", ".join(supported_dialects)
        msg = f"Unsupported source dialect provided for '--source-dialect': '{source_dialect}' (supported: {supported_dialects_description})"
        raise_validation_exception(msg)

    lineage_generator(engine, source_dialect, input_source, output_folder)


@lakebridge.command
def configure_secrets(*, w: WorkspaceClient) -> None:
    """Setup reconciliation connection profile details as Secrets on Databricks Workspace"""
    recon_conf = ReconConfigPrompts(w)

    # Prompt for source
    source = recon_conf.prompt_source()

    logger.info(f"Setting up Scope, Secrets for `{source}` reconciliation")
    recon_conf.prompt_and_save_connection_details()


@lakebridge.command
def configure_database_profiler(w: WorkspaceClient) -> None:
    """[Experimental] Installs and runs the Lakebridge Assessment package for database profiling"""
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "configure-profiler")
    prompts = ctx.prompts
    source_tech = prompts.choice("Select the source technology", PROFILER_SOURCE_SYSTEM).lower()
    ctx.add_user_agent_extra("profiler_source_tech", make_alphanum_or_semver(source_tech))
    user = ctx.current_user
    logger.debug(f"User: {user}")

    # Create appropriate assessment configurator
    assessment = create_assessment_configurator(source_system=source_tech, product_name="lakebridge", prompts=prompts)
    assessment.run()


@lakebridge.command
def install_transpile(
    *,
    w: WorkspaceClient,
    artifact: str | None = None,
    interactive: str | None = None,
    include_llm_transpiler: bool = False,
    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
) -> None:
    """Install or upgrade the Lakebridge transpilers."""
    # Avoid circular imports.
    from databricks.labs.lakebridge.install import installer  # pylint: disable=cyclic-import, import-outside-toplevel

    is_interactive = interactive_mode(interactive)
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "install-transpile")
    if artifact:
        ctx.add_user_agent_extra("artifact-overload", Path(artifact).name)
    # Internal: use LAKEBRIDGE_CLUSTER_TYPE=CLASSIC env var to use classic job cluster
    switch_use_serverless = os.environ.get("LAKEBRIDGE_CLUSTER_TYPE", "").upper() != "CLASSIC"
    if include_llm_transpiler:
        ctx.add_user_agent_extra("include-llm-transpiler", "true")
        # Decision was made not to prompt when include_llm_transpiler is set, and we expect users to use llm-transpile
        # and pass all the arguments.
        logger.info("Including LLM transpiler as part of install, interactive mode disabled: will skip questionnaire.")
        is_interactive = False

    user = w.current_user
    logger.debug(f"User: {user}")
    transpile_installer = installer(
        w,
        transpiler_repository,
        is_interactive=is_interactive,
        include_llm=include_llm_transpiler,
        switch_use_serverless=switch_use_serverless,
    )
    transpile_installer.run(module="transpile", artifact=artifact)


def interactive_mode(interactive: str | None, *, default: str = "auto", input_stream: TextIO = sys.stdin) -> bool:
    """Convert the raw '--interactive' argument into a boolean."""
    if interactive is None:
        interactive = default
    match interactive.lower():
        case "true":
            return True
        case "false":
            return False
        # Convention is that if the input_stream is a TTY, user interaction is allowed.
        case "auto":
            return input_stream.isatty()

    msg = f"Invalid value for '--interactive': {interactive!r} must be 'true', 'false' or 'auto'."
    raise_validation_exception(msg)


@lakebridge.command
def describe_transpile(
    *,
    w: WorkspaceClient,
    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
) -> None:
    """Describe the installed Lakebridge transpilers and available options."""
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "describe-transpile")
    user = w.current_user.me()
    logger.debug(f"User: {user}")
    transpilers_description = TranspilersDescription(transpiler_repository)
    json_description = transpilers_description.as_json()
    json.dump(json_description, sys.stdout, indent=2)


@lakebridge.command(is_unauthenticated=False)
def configure_reconcile(
    *,
    w: WorkspaceClient,
    transpiler_repository: TranspilerRepository = TranspilerRepository.user_home(),
) -> None:
    """Configure the Lakebridge reconciliation module"""
    # Avoid circular imports.
    from databricks.labs.lakebridge.install import installer  # pylint: disable=cyclic-import, import-outside-toplevel

    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "configure-reconcile")
    user = w.current_user
    logger.debug(f"User: {user}")
    if not w.config.warehouse_id:
        dbsql_id = _create_warehouse(w)
        w.config.warehouse_id = dbsql_id
    logger.debug(f"Warehouse ID used for configuring reconcile: {w.config.warehouse_id}.")
    reconcile_installer = installer(w, transpiler_repository, is_interactive=True)
    reconcile_installer.run(module="reconcile")


@lakebridge.command
def analyze(
    *,
    w: WorkspaceClient,
    source_directory: str | None = None,
    report_file: str | None = None,
    source_tech: str | None = None,
    generate_json: bool = False,
):
    """Run the Analyzer"""
    ctx = ApplicationContext(w)
    try:
        result = ctx.analyzer.run_analyzer(source_directory, report_file, source_tech, generate_json)
        ctx.add_user_agent_extra("analyzer_source_tech", result.source_system)
    finally:
        exception_cls, _, _ = sys.exc_info()
        if exception_cls is not None:
            ctx.add_user_agent_extra("analyzer_error", exception_cls.__name__)

        ctx.add_user_agent_extra("cmd", "analyze")
        logger.debug(f"User: {ctx.current_user}")


def _validate_llm_transpile_args(
    input_source: str | None,
    output_ws_folder: str | None,
    source_dialect: str | None,
    prompts: Prompts,
) -> tuple[str, str, str]:

    _switch_dialects = get_switch_dialects()

    # Validate presence after attempting to source from config
    if not input_source:
        input_source = prompts.question("Enter input SQL path")
    if not output_ws_folder:
        output_ws_folder = prompts.question("Enter output workspace folder must start with /Workspace/")
    if not source_dialect:
        source_dialect = prompts.choice("Select the source dialect", sorted(_switch_dialects))

    # Validate input_source path exists (local path)
    if not Path(input_source).exists():
        raise_validation_exception(f"Invalid path for '--input-source': Path '{input_source}' does not exist.")

    # Validate output_ws_folder is a workspace path
    if not str(output_ws_folder).startswith("/Workspace/"):
        raise_validation_exception(
            f"Invalid value for '--output-ws-folder': workspace output path must start with /Workspace/. Got: {output_ws_folder!r}"
        )

    if source_dialect not in _switch_dialects:
        raise_validation_exception(
            f"Invalid value for '--source-dialect': {source_dialect!r} must be one of: {', '.join(sorted(_switch_dialects))}"
        )

    return input_source, output_ws_folder, source_dialect


@lakebridge.command
def llm_transpile(
    *,
    w: WorkspaceClient,
    accept_terms: bool = False,
    input_source: str | None = None,
    output_ws_folder: str | None = None,
    source_dialect: str | None = None,
    catalog_name: str | None = None,
    schema_name: str | None = None,
    volume: str | None = None,
    foundation_model: str | None = None,
    ctx: ApplicationContext | None = None,
) -> None:
    """Transpile source code to Databricks using LLM Transpiler (Switch)"""
    if ctx is None:
        ctx = ApplicationContext(w)
    del w
    ctx.add_user_agent_extra("cmd", "llm-transpile")
    user = ctx.current_user
    logger.debug(f"User: {user}")

    if not accept_terms:
        logger.warning(
            """Please read and accept these terms before proceeding:
    This feature leverages a Large Language Model (LLM) to analyse and convert
    your provided content, code and data. You consent to your content being
    transmitted to, processed by, and returned from the foundation models hosted
    by Databricks or external foundation models you have configured in your
    workspace. The outputs of the LLM are generated automatically without human
    review, and may contain inaccuracies or errors. You are responsible for
    reviewing and validating all outputs before relying on them for any critical
    or production use.

    By using this feature you accept these terms, re-run with '--accept-terms=true'.
                """
        )
        raise SystemExit("LLM transpiler terms not accepted, exiting.")

    prompts = ctx.prompts
    resource_configurator = ctx.resource_configurator

    # If CLI args are missing, try to read them from config.yml
    input_source, output_ws_folder, source_dialect = _validate_llm_transpile_args(
        input_source,
        output_ws_folder,
        source_dialect,
        prompts,
    )

    if catalog_name is None:
        catalog_name = resource_configurator.prompt_for_catalog_setup(default_catalog_name="lakebridge")

    if schema_name is None:
        schema_name = resource_configurator.prompt_for_schema_setup(catalog=catalog_name, default_schema_name="switch")

    if volume is None:
        volume = resource_configurator.prompt_for_volume_setup(
            catalog=catalog_name, schema=schema_name, default_volume_name="switch_volume"
        )

    resource_configurator.has_necessary_access(catalog_name, schema_name, volume)

    if foundation_model is None:
        foundation_model = resource_configurator.prompt_for_foundation_model_choice()

    job_list = ctx.install_state.jobs
    if "Switch" not in job_list:
        logger.debug(f"Missing Switch from installed state jobs: {job_list!r}")
        raise RuntimeError(
            "Switch Job not found. "
            "Please run 'databricks labs lakebridge install-transpile --include-llm-transpiler true' first."
        )
    job_id = int(job_list["Switch"])
    logger.debug(f"Switch job ID found: {job_id}")

    ctx.add_user_agent_extra("transpiler_source_dialect", source_dialect)
    job_runner = SwitchRunner(ctx.workspace_client)
    volume_input_path = job_runner.upload_to_volume(
        local_path=Path(input_source),
        catalog=catalog_name,
        schema=schema_name,
        volume=volume,
    )

    job_runner.run(
        volume_input_path=volume_input_path,
        output_ws_folder=output_ws_folder,
        source_tech=source_dialect,
        catalog=catalog_name,
        schema=schema_name,
        foundation_model=foundation_model,
        job_id=job_id,
    )


@lakebridge.command()
def execute_database_profiler(w: WorkspaceClient, source_tech: str | None = None) -> None:
    """Execute the Profiler Extraction for the given source technology"""
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "execute-profiler")
    prompts = ctx.prompts
    if source_tech is None:
        source_tech = prompts.choice("Select the source technology", PROFILER_SOURCE_SYSTEM)
    source_tech = source_tech.lower()

    if source_tech not in PROFILER_SOURCE_SYSTEM:
        logger.error(f"Only the following source systems are supported: {PROFILER_SOURCE_SYSTEM}")
        raise_validation_exception(f"Invalid source technology {source_tech}")

    ctx.add_user_agent_extra("profiler_source_tech", make_alphanum_or_semver(source_tech))
    user = ctx.current_user
    logger.debug(f"User: {user}")
    # check if cred_file is present which has the connection details before running the profiler
    file = cred_file(PRODUCT_NAME)
    if not file.exists():
        raise_validation_exception(
            f"Connection details not found. Please run `databricks labs lakebridge configure-database-profiler` "
            f"to set up connection details for {source_tech}."
        )
    profiler = Profiler.create(source_tech)

    # TODO: Add extractor logic to ApplicationContext instead of creating inside the Profiler class
    profiler.profile()


@lakebridge.command()
def create_profiler_dashboard(
    *,
    w: WorkspaceClient,
    extract_file: str,
    source_tech: str,
    volume_path: str,
    catalog_name: str,
    schema_name: str,
) -> None:
    """Deploys a profiler summary as a Databricks dashboard"""
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "create-profiler-dashboard")
    ctx.dashboard_manager.upload_duckdb_to_uc_volume(extract_file, volume_path)
    ctx.dashboard_manager.create_profiler_summary_dashboard(source_tech, catalog_name, schema_name)


def _test_database_connection(source_tech: str, raw_config: dict) -> None:
    """Test connection to the source database with appropriate error handling."""
    # Handle synapse-specific validation using dedicated helper
    if source_tech == "synapse":
        validate_synapse_pools(raw_config)
        logger.info("Connection to the source system successful")
        return

    # For other source technologies, use DatabaseManager directly
    with DatabaseManager(source_tech, raw_config) as db_manager:
        response = db_manager.check_connection()
    logger.debug(f"Connection response: {response}")
    logger.info("Connection to the source system successful")


@lakebridge.command()
def test_profiler_connection(
    *,
    w: WorkspaceClient,
    source_tech: str | None = None,
    cred_file_path: str | None = None,
) -> None:
    """[Internal] Test the connection to the source database for profiling"""
    ctx = ApplicationContext(w)
    ctx.add_user_agent_extra("cmd", "test-profiler-connection")
    prompts = ctx.prompts

    source_tech = (
        source_tech.lower()
        if source_tech
        else prompts.choice("Select the source technology", PROFILER_SOURCE_SYSTEM).lower()
    )

    if source_tech not in PROFILER_SOURCE_SYSTEM:
        logger.error(f"Only the following source systems are supported: {PROFILER_SOURCE_SYSTEM}")
        raise_validation_exception(f"Invalid source technology {source_tech}")

    ctx.add_user_agent_extra("profiler_source_tech", make_alphanum_or_semver(source_tech))
    logger.debug(f"User: {ctx.current_user}")

    # Use provided credential file path or fall back to default
    credential_file = Path(cred_file_path) if cred_file_path else cred_file(PRODUCT_NAME)

    # Check if credential file exists
    if not credential_file.exists():
        raise_validation_exception(
            f"Connection details not found. Please run `databricks labs lakebridge configure-database-profiler` "
            f"to set up connection details for {source_tech}."
        )

    logger.info(f"Testing connection for source technology: {source_tech}")

    cred_manager = create_credential_manager(PRODUCT_NAME, EnvGetter(), creds_path=credential_file)

    try:
        raw_config = cred_manager.get_credentials(source_tech)
    except KeyError as e:
        logger.error(f"Credential configuration error: {e}")
        logger.fatal(
            f"Invalid credentials for {source_tech}. Please run `databricks labs lakebridge configure-database-profiler`."
        )
        return

    try:
        _test_database_connection(source_tech, raw_config)
    except ConnectionError as e:
        logger.error(f"Failed to connect to the source system: {e}")
        error_msg = str(e).lower()
        if any(pattern in error_msg for pattern in ("im002", "odbc driver not found", "can't open lib")):
            logger.fatal("Missing ODBC driver, Please install pre-req. Exiting...")
        else:
            logger.fatal("Connection validation failed. Exiting...")
    except Exception as e:  # pylint: disable=broad-exception-caught
        # Catch all exceptions to provide user-friendly error messages for CLI
        logger.error(f"Unexpected error during connection test: {e}")
        logger.fatal("Connection test failed. Exiting...")


if __name__ == "__main__":
    lakebridge()
