"""
Orthogonality Analysis Storage Backend

This module handles storage and retrieval of orthogonality analysis results,
including both ClickHouse tables and file-based matrix storage.

Key optimization: Batch loading of factor values using ClickHouse JOINs
instead of sequential table reads.
"""

import os
import gc
import json
import logging
from pathlib import Path
from typing import Optional, Dict, Any, List, Tuple
from datetime import datetime
import pandas as pd
import numpy as np

from ..core.config import FactorDBConfig, AssetType, FactorType
from .schema import (
    get_create_table_sql,
    get_orthogonality_schemas,
    ORTHOGONALITY_ANALYSIS_SCHEMA,
    HIGH_CORRELATION_PAIRS_SCHEMA,
    FACTOR_CLUSTERS_SCHEMA,
    MST_EDGES_SCHEMA,
    VIF_RESULTS_SCHEMA,
    FACTOR_SELECTION_SCHEMA,
    EIGENVALUE_DISTRIBUTION_SCHEMA,
)

logger = logging.getLogger(__name__)


class OrthogonalityStorage:
    """
    Storage backend for orthogonality analysis results.
    
    Handles:
    - ClickHouse tables for structured results
    - File storage (Parquet/NPY) for large matrices
    - Batch loading of factor values with optimized JOINs
    """
    
    def __init__(
        self,
        config: Optional[FactorDBConfig] = None,
        data_dir: Optional[str] = None
    ):
        """
        Initialize orthogonality storage.
        
        Args:
            config: FactorDB configuration
            data_dir: Directory for file-based storage (matrices, etc.)
        """
        self.config = config or FactorDBConfig()
        self.data_dir = Path(data_dir or "data/orthogonality")
        self._db_cache: Dict[str, Any] = {}
        self._initialized_dbs = set()
    
    def close(self) -> None:
        """Close all cached database connections."""
        for db_name, db in self._db_cache.items():
            try:
                db.close()
            except Exception as e:
                logger.warning(f"Failed to close connection to {db_name}: {e}")
        self._db_cache.clear()
    
    def __enter__(self):
        return self
    
    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()
        return False
    
    def _get_db(
        self,
        asset_type: AssetType,
        factor_type: FactorType
    ):
        """Get or create database connection."""
        db_name = self.config.get_factor_db_name(asset_type, factor_type)
        
        if db_name in self._db_cache:
            return self._db_cache[db_name], db_name
        
        try:
            from quantchdb import ClickHouseDatabase
        except ImportError as e:
            raise ImportError(f"quantchdb required: {e}")
        
        db = ClickHouseDatabase(
            config=self.config.get_db_config(asset_type, factor_type),
            terminal_log=False,
            file_log=False
        )
        self._db_cache[db_name] = db
        return db, db_name
    
    def initialize_orthogonality_tables(
        self,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Create orthogonality analysis tables if they don't exist."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        db_key = f"orth_{asset_type.value}_{factor_type.value}"
        if db_key in self._initialized_dbs:
            return
        
        schemas = get_orthogonality_schemas()
        for schema_name, schema in schemas.items():
            sql = get_create_table_sql(schema, database=db_name)
            db.execute(sql)
            logger.debug(f"Ensured table {db_name}.{schema['table_name']}")
        
        self._initialized_dbs.add(db_key)
        logger.info(f"Initialized orthogonality tables in {db_name}")
    
    # =========================================================================
    # Batch Factor Loading (Optimized)
    # =========================================================================
    
    def load_factor_matrix_batch(
        self,
        factor_ids: List[str],
        asset_type: AssetType,
        factor_type: FactorType,
        start_date: Optional[str] = None,
        end_date: Optional[str] = None,
        batch_size: int = 20
    ) -> pd.DataFrame:
        """
        Load multiple factor values efficiently using batch JOINs.
        
        Instead of reading each factor table sequentially, this method:
        1. Splits factors into batches
        2. Uses UNION ALL to combine factor tables in a single query
        3. Returns a wide-format matrix for correlation analysis
        
        Args:
            factor_ids: List of factor IDs to load
            asset_type: Asset type
            factor_type: Factor type
            start_date: Optional start date filter
            end_date: Optional end date filter
            batch_size: Number of factors per batch query
            
        Returns:
            Wide-format DataFrame with multi-index (date, code) and factors as columns
        """
        if not factor_ids:
            return pd.DataFrame()
        
        db, db_name = self._get_db(asset_type, factor_type)
        
        # Build date filter
        date_conditions = []
        if start_date:
            date_conditions.append(f"date >= '{start_date}'")
        if end_date:
            date_conditions.append(f"date <= '{end_date}'")
        date_where = " AND ".join(date_conditions) if date_conditions else "1=1"

        logger.info(f"  Date filter: {date_where}")

        all_data = []
        total_rows = 0
        
        # Process in batches to avoid query size limits
        for i in range(0, len(factor_ids), batch_size):
            batch_factors = factor_ids[i:i + batch_size]
            
            # Build UNION ALL query for batch
            union_parts = []
            for fid in batch_factors:
                # Check if table exists first
                table_check = db.fetch(
                    f"SELECT name FROM system.tables "
                    f"WHERE database = '{db_name}' AND name = '{fid}'"
                )
                if len(table_check) == 0:
                    logger.warning(f"Table {fid} not found, skipping")
                    continue
                
                union_parts.append(f"""
                    SELECT 
                        code, 
                        date, 
                        factor_value,
                        '{fid}' as factor_id
                    FROM {db_name}.{fid}
                    WHERE {date_where}
                """)
            
            if not union_parts:
                continue
            
            # Execute batch query
            batch_sql = " UNION ALL ".join(union_parts)
            batch_df = db.fetch(batch_sql)
            
            if len(batch_df) > 0:
                all_data.append(batch_df)
                total_rows += len(batch_df)

            # Memory optimization: garbage collection after each batch
            gc.collect()

            logger.info(f"  Loaded batch {i//batch_size + 1}/{(len(factor_ids) + batch_size - 1)//batch_size}: "
                        f"{len(batch_factors)} factors, {len(batch_df) if len(batch_df) > 0 else 0:,} rows")
        
        if not all_data:
            return pd.DataFrame()

        logger.info(f"  Total rows loaded: {total_rows:,}")

        # Combine all batches
        long_df = pd.concat(all_data, ignore_index=True)

        # Free memory from individual batches
        del all_data
        gc.collect()

        # Convert to float32 for memory efficiency
        long_df['factor_value'] = long_df['factor_value'].astype(np.float32)

        # Pivot to wide format: rows = (date, code), columns = factor_id
        wide_df = long_df.pivot_table(
            index=['date', 'code'],
            columns='factor_id',
            values='factor_value',
            aggfunc='first'
        )

        # Free memory from long format
        del long_df
        gc.collect()

        # Reorder columns to match input order
        available_factors = [f for f in factor_ids if f in wide_df.columns]
        wide_df = wide_df[available_factors]

        logger.info(f"Loaded factor matrix: {wide_df.shape[0]} obs x {wide_df.shape[1]} factors")

        return wide_df

    def get_available_dates(
        self,
        factor_ids: List[str],
        asset_type: AssetType,
        factor_type: FactorType,
        start_date: Optional[str] = None,
        end_date: Optional[str] = None
    ) -> List[str]:
        """
        Get list of available dates for factors.

        Args:
            factor_ids: List of factor IDs
            asset_type: Asset type
            factor_type: Factor type
            start_date: Optional start date filter
            end_date: Optional end date filter

        Returns:
            List of date strings sorted ascending
        """
        if not factor_ids:
            return []

        db, db_name = self._get_db(asset_type, factor_type)

        # Build date filter
        date_conditions = []
        if start_date:
            date_conditions.append(f"date >= '{start_date}'")
        if end_date:
            date_conditions.append(f"date <= '{end_date}'")
        date_where = " AND ".join(date_conditions) if date_conditions else "1=1"

        # Use first factor to get dates (assume all factors have same dates)
        first_factor = factor_ids[0]

        # Check table exists
        table_check = db.fetch(
            f"SELECT name FROM system.tables "
            f"WHERE database = '{db_name}' AND name = '{first_factor}'"
        )
        if len(table_check) == 0:
            logger.warning(f"Table {first_factor} not found")
            return []

        sql = f"""
            SELECT DISTINCT date
            FROM {db_name}.{first_factor}
            WHERE {date_where}
            ORDER BY date
        """
        result = db.fetch(sql)

        if len(result) == 0:
            return []

        dates = [str(row[0]) for row in result.values]
        logger.info(f"  Found {len(dates)} dates in range")
        return dates

    def stream_factor_data_by_date(
        self,
        factor_ids: List[str],
        asset_type: AssetType,
        factor_type: FactorType,
        dates: List[str]
    ):
        """
        Stream factor data one date at a time using numpy arrays (ultra memory efficient).

        This generator yields cross-sectional data for each date as a numpy array,
        keeping peak memory usage minimal.

        Args:
            factor_ids: List of factor IDs
            asset_type: Asset type
            factor_type: Factor type
            dates: List of dates to process

        Yields:
            Tuple of (date_str, numpy_array, n_stocks) where array shape is (n_stocks, n_factors)
        """
        if not factor_ids or not dates:
            return

        db, db_name = self._get_db(asset_type, factor_type)

        # Verify which tables exist (do this once upfront)
        valid_factors = []
        for fid in factor_ids:
            table_check = db.fetch(
                f"SELECT name FROM system.tables "
                f"WHERE database = '{db_name}' AND name = '{fid}'"
            )
            if len(table_check) > 0:
                valid_factors.append(fid)
            else:
                logger.warning(f"Table {fid} not found, skipping")

        if not valid_factors:
            return

        n_factors = len(valid_factors)
        factor_to_idx = {f: i for i, f in enumerate(valid_factors)}

        logger.info(f"  Streaming {len(dates)} dates × {n_factors} factors (numpy mode)")

        for date_idx, date_str in enumerate(dates):
            # Build UNION ALL query for single date (all factors)
            union_parts = []
            for fid in valid_factors:
                union_parts.append(f"""
                    SELECT
                        code,
                        factor_value,
                        '{fid}' as factor_id
                    FROM {db_name}.{fid}
                    WHERE date = '{date_str}'
                """)

            sql = " UNION ALL ".join(union_parts)
            result = db.fetch(sql)

            if len(result) == 0:
                continue

            # Convert to numpy efficiently
            # Group by code and create dense matrix
            codes = result['code'].values
            values = result['factor_value'].values.astype(np.float32)
            factor_names = result['factor_id'].values

            # Get unique codes for this date
            unique_codes = np.unique(codes)
            n_stocks = len(unique_codes)

            if n_stocks < 30:  # Skip dates with too few stocks
                continue

            # Create code to index mapping
            code_to_idx = {c: i for i, c in enumerate(unique_codes)}

            # Create numpy array (stocks × factors), fill with NaN
            data_matrix = np.full((n_stocks, n_factors), np.nan, dtype=np.float32)

            # Fill in values
            for i in range(len(codes)):
                stock_idx = code_to_idx[codes[i]]
                factor_idx = factor_to_idx[factor_names[i]]
                data_matrix[stock_idx, factor_idx] = values[i]

            # Free intermediate data
            del result, codes, values, factor_names, unique_codes, code_to_idx

            if (date_idx + 1) % 100 == 0:
                logger.info(f"    Processed {date_idx + 1}/{len(dates)} dates")

            yield date_str, data_matrix, n_stocks

    def load_factor_ic_values(
        self,
        factor_ids: List[str],
        asset_type: AssetType,
        factor_type: FactorType,
        ic_column: str = "rank_ic_mean"
    ) -> Dict[str, float]:
        """
        Load IC values for multiple factors in one query.
        
        Args:
            factor_ids: List of factor IDs
            asset_type: Asset type
            factor_type: Factor type
            ic_column: Which IC column to use
            
        Returns:
            Dictionary mapping factor_id -> IC value
        """
        if not factor_ids:
            return {}
        
        db, db_name = self._get_db(asset_type, factor_type)
        
        # Build IN clause
        ids_str = ", ".join(f"'{fid}'" for fid in factor_ids)
        
        sql = f"""
            SELECT factor_id, {ic_column} as ic_value
            FROM {db_name}.A2_factor_evaluate
            WHERE factor_id IN ({ids_str})
        """
        
        result = db.fetch(sql)
        
        ic_dict = {}
        for _, row in result.iterrows():
            fid = row['factor_id']
            ic = row['ic_value']
            ic_dict[fid] = float(ic) if pd.notna(ic) else 0.0
        
        # Fill missing with 0
        for fid in factor_ids:
            if fid not in ic_dict:
                ic_dict[fid] = 0.0
        
        return ic_dict
    
    # =========================================================================
    # Analysis ID Generation
    # =========================================================================
    
    def generate_analysis_id(
        self,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> str:
        """Generate unique analysis ID."""
        prefix = "OA"
        asset_code = "stk" if asset_type == AssetType.STOCK else "etf"
        date_str = datetime.now().strftime("%Y%m%d")
        
        db, db_name = self._get_db(asset_type, factor_type)
        
        # Find next sequence number for today
        pattern = f"{prefix}_{asset_code}_{date_str}_%"
        sql = f"""
            SELECT analysis_id
            FROM {db_name}.B1_orthogonality_analysis
            WHERE analysis_id LIKE '{pattern}'
            ORDER BY analysis_id DESC
            LIMIT 1
        """
        
        try:
            result = db.fetch(sql)
            if len(result) > 0:
                last_id = result.iloc[0]['analysis_id']
                seq = int(last_id.split('_')[-1]) + 1
            else:
                seq = 1
        except Exception:
            seq = 1
        
        return f"{prefix}_{asset_code}_{date_str}_{seq:03d}"
    
    # =========================================================================
    # Save Analysis Results
    # =========================================================================
    
    def save_analysis_summary(
        self,
        analysis_id: str,
        report: Any,  # OrthogonalityReport
        asset_type: AssetType,
        factor_type: FactorType,
        data_start_date: str,
        data_end_date: str,
        params: Dict[str, Any]
    ) -> None:
        """Save main analysis summary to B1_orthogonality_analysis."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        summary = report.summary
        
        # Prepare file paths
        analysis_dir = self._get_analysis_dir(analysis_id, asset_type)
        corr_path = str(analysis_dir / "correlation_matrix.parquet")
        dist_path = str(analysis_dir / "distance_matrix.parquet")
        eigen_path = str(analysis_dir / "eigenvalues.npy")
        linkage_path = str(analysis_dir / "linkage_matrix.npy")
        
        sql = f"""
            INSERT INTO {db_name}.B1_orthogonality_analysis
            (analysis_id, asset_type, factor_type, analysis_datetime,
             total_factors, data_start_date, data_end_date,
             effective_n_90, effective_n_95, mean_correlation, max_correlation, min_correlation,
             redundancy_ratio, high_corr_pairs_count,
             n_clusters, linkage_method, n_central_factors, n_peripheral_factors, mst_total_distance,
             n_selected, n_removed, selection_ratio, vif_threshold, marginal_ic_threshold,
             correlation_matrix_path, distance_matrix_path, eigenvalues_path, linkage_matrix_path)
            VALUES
            ('{analysis_id}', '{asset_type.value}', '{factor_type.value}', now64(3),
             {summary['total_factors']}, '{data_start_date}', '{data_end_date}',
             {summary['effective_n_90']}, {summary['effective_n_95']}, 
             {summary['mean_correlation']}, {summary['max_correlation']}, 
             {report.correlation_stats.min_correlation},
             {summary['redundancy_ratio']}, {summary['high_correlation_pairs']},
             {summary['n_clusters']}, '{params.get("linkage_method", "ward")}',
             {summary['n_central_factors']}, {summary['n_peripheral_factors']},
             {report.mst_result.total_distance if report.mst_result else 0},
             {summary['n_selected']}, {summary['n_removed']}, {summary['selection_ratio']},
             {params.get('vif_threshold', 5.0)}, {params.get('marginal_ic_threshold', 0.015)},
             '{corr_path}', '{dist_path}', '{eigen_path}', '{linkage_path}')
        """
        
        db.execute(sql)
        logger.info(f"Saved analysis summary: {analysis_id}")
    
    def save_high_correlation_pairs(
        self,
        analysis_id: str,
        pairs: List[Tuple[str, str, float]],
        cluster_labels: Dict[str, int],
        distance_matrix: pd.DataFrame,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save high correlation pairs to B2_high_correlation_pairs."""
        if not pairs:
            return
        
        db, db_name = self._get_db(asset_type, factor_type)
        
        values_list = []
        for f1, f2, corr in pairs:
            dist = distance_matrix.loc[f1, f2] if f1 in distance_matrix.index and f2 in distance_matrix.columns else 0
            c1 = cluster_labels.get(f1, 0)
            c2 = cluster_labels.get(f2, 0)
            same = 1 if c1 == c2 else 0
            cluster_id = c1 if same else "NULL"
            
            values_list.append(
                f"('{analysis_id}', '{f1}', '{f2}', {corr}, {dist}, {same}, {cluster_id})"
            )
        
        # Insert in batches
        batch_size = 1000
        for i in range(0, len(values_list), batch_size):
            batch = values_list[i:i + batch_size]
            sql = f"""
                INSERT INTO {db_name}.B2_high_correlation_pairs
                (analysis_id, factor_id_1, factor_id_2, correlation, distance, same_cluster, cluster_id)
                VALUES {', '.join(batch)}
            """
            db.execute(sql)
        
        logger.info(f"Saved {len(pairs)} high correlation pairs")
    
    def save_factor_clusters(
        self,
        analysis_id: str,
        clustering_result: Any,  # HierarchicalClusteringResult
        mst_result: Any,  # MSTResult
        representative_factors: Dict[int, str],
        ic_values: Dict[str, float],
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save factor cluster assignments to B3_factor_clusters."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        # Build cluster size map
        cluster_sizes = {}
        cluster_corrs = {}
        for cluster in clustering_result.clusters:
            cluster_sizes[cluster.cluster_id] = cluster.size
            cluster_corrs[cluster.cluster_id] = cluster.mean_intra_correlation
        
        # Build MST degree map
        mst_degrees = {}
        for edge in mst_result.edges:
            mst_degrees[edge.source] = mst_degrees.get(edge.source, 0) + 1
            mst_degrees[edge.target] = mst_degrees.get(edge.target, 0) + 1
        
        rep_factors = set(representative_factors.values())
        central_set = set(mst_result.central_factors)
        peripheral_set = set(mst_result.peripheral_factors)
        
        values_list = []
        for factor_id, cluster_id in clustering_result.cluster_labels.items():
            ic = ic_values.get(factor_id, 0)
            rank_ic = ic_values.get(factor_id, 0)  # Same for now
            
            values_list.append(
                f"('{analysis_id}', '{factor_id}', {cluster_id}, "
                f"{cluster_sizes.get(cluster_id, 0)}, "
                f"{cluster_corrs.get(cluster_id, 0)}, "
                f"{1 if factor_id in rep_factors else 0}, "
                f"{mst_degrees.get(factor_id, 0)}, "
                f"{mst_result.centrality_scores.get(factor_id, 0)}, "
                f"{1 if factor_id in central_set else 0}, "
                f"{1 if factor_id in peripheral_set else 0}, "
                f"{ic}, {rank_ic})"
            )
        
        # Insert in batches
        batch_size = 500
        for i in range(0, len(values_list), batch_size):
            batch = values_list[i:i + batch_size]
            sql = f"""
                INSERT INTO {db_name}.B3_factor_clusters
                (analysis_id, factor_id, cluster_id, cluster_size, intra_cluster_corr,
                 is_representative, mst_degree, centrality_score, is_central, is_peripheral,
                 factor_ic, factor_rank_ic)
                VALUES {', '.join(batch)}
            """
            db.execute(sql)
        
        logger.info(f"Saved {len(values_list)} factor cluster assignments")
    
    def save_mst_edges(
        self,
        analysis_id: str,
        mst_result: Any,  # MSTResult
        cluster_labels: Dict[str, int],
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save MST edges to B4_mst_edges."""
        if not mst_result.edges:
            return
        
        db, db_name = self._get_db(asset_type, factor_type)
        
        values_list = []
        for idx, edge in enumerate(mst_result.edges):
            src_cluster = cluster_labels.get(edge.source, 0)
            tgt_cluster = cluster_labels.get(edge.target, 0)
            is_inter = 1 if src_cluster != tgt_cluster else 0
            
            values_list.append(
                f"('{analysis_id}', {idx}, '{edge.source}', '{edge.target}', "
                f"{edge.distance}, {edge.correlation}, "
                f"{src_cluster}, {tgt_cluster}, {is_inter})"
            )
        
        sql = f"""
            INSERT INTO {db_name}.B4_mst_edges
            (analysis_id, edge_id, source_factor, target_factor, distance, correlation,
             source_cluster, target_cluster, is_inter_cluster)
            VALUES {', '.join(values_list)}
        """
        db.execute(sql)
        logger.info(f"Saved {len(mst_result.edges)} MST edges")
    
    def save_vif_results(
        self,
        analysis_id: str,
        vif_results: Dict[int, Any],  # Dict[cluster_id, ClusterVIFResult]
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save VIF results to B5_vif_results."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        values_list = []
        for cluster_id, cluster_result in vif_results.items():
            for vif_res in cluster_result.factor_vifs:
                action = "remove" if vif_res.factor_id in cluster_result.recommended_remove else "keep"
                vif_val = min(vif_res.vif, 999999)  # Cap inf values
                
                values_list.append(
                    f"('{analysis_id}', '{vif_res.factor_id}', {cluster_id}, "
                    f"{vif_val}, {vif_res.r_squared}, "
                    f"{1 if vif_res.is_collinear else 0}, '{action}')"
                )
        
        if not values_list:
            return
        
        # Insert in batches
        batch_size = 500
        for i in range(0, len(values_list), batch_size):
            batch = values_list[i:i + batch_size]
            sql = f"""
                INSERT INTO {db_name}.B5_vif_results
                (analysis_id, factor_id, cluster_id, vif, r_squared, is_collinear, recommended_action)
                VALUES {', '.join(batch)}
            """
            db.execute(sql)
        
        logger.info(f"Saved {len(values_list)} VIF results")
    
    def save_factor_selection(
        self,
        analysis_id: str,
        selection_result: Any,  # SelectionResult
        cluster_labels: Dict[str, int],
        representative_factors: Dict[int, str],
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save factor selection results to B6_factor_selection."""
        db, db_name = self._get_db(asset_type, factor_type)

        rep_factors = set(representative_factors.values())
        selected_set = set(selection_result.selected_factors)

        values_list = []
        for factor_id, marginal_result in selection_result.marginal_ics.items():
            cluster_id = cluster_labels.get(factor_id, 0)
            is_selected = 1 if factor_id in selected_set else 0
            order = marginal_result.order_added if marginal_result.order_added else 0

            # Determine removal reason
            if not is_selected:
                if marginal_result.marginal_ic < 0.015:
                    reason = "'Low marginal IC'"
                else:
                    reason = "'Redundant'"
            else:
                reason = "''"

            values_list.append(
                f"('{analysis_id}', '{factor_id}', "
                f"{marginal_result.original_ic}, {marginal_result.original_ic}, "
                f"{marginal_result.marginal_ic}, {marginal_result.ic_contribution}, "
                f"{1 if marginal_result.is_significant else 0}, "
                f"{is_selected}, {order}, {reason}, "
                f"{cluster_id}, {1 if factor_id in rep_factors else 0})"
            )
        
        if not values_list:
            return
        
        # Insert in batches
        batch_size = 500
        for i in range(0, len(values_list), batch_size):
            batch = values_list[i:i + batch_size]
            sql = f"""
                INSERT INTO {db_name}.B6_factor_selection
                (analysis_id, factor_id, original_ic, original_rank_ic,
                 marginal_ic, ic_contribution, is_significant,
                 is_selected, selection_order, removal_reason,
                 cluster_id, is_cluster_rep)
                VALUES {', '.join(batch)}
            """
            db.execute(sql)
        
        logger.info(f"Saved {len(values_list)} factor selection results")
    
    def save_eigenvalue_distribution(
        self,
        analysis_id: str,
        effective_n: Any,  # EffectiveFactorResult
        asset_type: AssetType,
        factor_type: FactorType
    ) -> None:
        """Save eigenvalue distribution to B7_eigenvalue_distribution."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        values_list = []
        for i, (ev, evr, cvr) in enumerate(zip(
            effective_n.eigenvalues,
            effective_n.explained_variance_ratio,
            effective_n.cumulative_variance_ratio
        )):
            values_list.append(
                f"('{analysis_id}', {i + 1}, {ev}, {evr}, {cvr})"
            )
        
        # Insert in batches
        batch_size = 500
        for i in range(0, len(values_list), batch_size):
            batch = values_list[i:i + batch_size]
            sql = f"""
                INSERT INTO {db_name}.B7_eigenvalue_distribution
                (analysis_id, component_index, eigenvalue, explained_variance_ratio, cumulative_variance_ratio)
                VALUES {', '.join(batch)}
            """
            db.execute(sql)
        
        logger.info(f"Saved {len(values_list)} eigenvalue records")
    
    # =========================================================================
    # File Storage
    # =========================================================================
    
    def _get_analysis_dir(self, analysis_id: str, asset_type: AssetType) -> Path:
        """Get directory for analysis file storage."""
        analysis_dir = self.data_dir / asset_type.value.lower() / analysis_id
        analysis_dir.mkdir(parents=True, exist_ok=True)
        return analysis_dir
    
    def save_matrices(
        self,
        analysis_id: str,
        correlation_matrix: pd.DataFrame,
        distance_matrix: pd.DataFrame,
        eigenvalues: np.ndarray,
        linkage_matrix: np.ndarray,
        asset_type: AssetType
    ) -> Dict[str, str]:
        """Save large matrices to files."""
        analysis_dir = self._get_analysis_dir(analysis_id, asset_type)
        
        paths = {}
        
        # Save correlation matrix (sparse format - upper triangle only)
        corr_path = analysis_dir / "correlation_matrix.parquet"
        corr_long = self._matrix_to_sparse(correlation_matrix, "correlation")
        corr_long.to_parquet(corr_path, compression='zstd')
        paths['correlation'] = str(corr_path)
        
        # Save distance matrix (sparse format)
        dist_path = analysis_dir / "distance_matrix.parquet"
        dist_long = self._matrix_to_sparse(distance_matrix, "distance")
        dist_long.to_parquet(dist_path, compression='zstd')
        paths['distance'] = str(dist_path)
        
        # Save eigenvalues
        eigen_path = analysis_dir / "eigenvalues.npy"
        np.save(eigen_path, eigenvalues)
        paths['eigenvalues'] = str(eigen_path)
        
        # Save linkage matrix
        linkage_path = analysis_dir / "linkage_matrix.npy"
        np.save(linkage_path, linkage_matrix)
        paths['linkage'] = str(linkage_path)
        
        # Save metadata
        metadata = {
            "analysis_id": analysis_id,
            "created_at": datetime.now().isoformat(),
            "factor_order": list(correlation_matrix.columns),
            "n_factors": len(correlation_matrix.columns),
        }
        with open(analysis_dir / "metadata.json", 'w') as f:
            json.dump(metadata, f, indent=2)
        
        logger.info(f"Saved matrices to {analysis_dir}")
        return paths
    
    def _matrix_to_sparse(self, matrix: pd.DataFrame, value_col: str) -> pd.DataFrame:
        """Convert symmetric matrix to sparse (upper triangle) format."""
        factors = matrix.columns.tolist()
        rows = []
        
        for i, f1 in enumerate(factors):
            for j, f2 in enumerate(factors):
                if i <= j:  # Upper triangle including diagonal
                    rows.append({
                        'factor_i': f1,
                        'factor_j': f2,
                        value_col: matrix.iloc[i, j]
                    })
        
        return pd.DataFrame(rows)
    
    def load_correlation_matrix(
        self,
        analysis_id: str,
        asset_type: AssetType
    ) -> pd.DataFrame:
        """Load correlation matrix from file."""
        analysis_dir = self._get_analysis_dir(analysis_id, asset_type)
        corr_path = analysis_dir / "correlation_matrix.parquet"
        
        if not corr_path.exists():
            raise FileNotFoundError(f"Correlation matrix not found: {corr_path}")
        
        sparse_df = pd.read_parquet(corr_path)
        return self._sparse_to_matrix(sparse_df, "correlation")
    
    def _sparse_to_matrix(self, sparse_df: pd.DataFrame, value_col: str) -> pd.DataFrame:
        """Convert sparse format back to symmetric matrix."""
        factors = sorted(set(sparse_df['factor_i'].tolist() + sparse_df['factor_j'].tolist()))
        n = len(factors)
        factor_idx = {f: i for i, f in enumerate(factors)}
        
        matrix = np.zeros((n, n))
        
        for _, row in sparse_df.iterrows():
            i = factor_idx[row['factor_i']]
            j = factor_idx[row['factor_j']]
            val = row[value_col]
            matrix[i, j] = val
            matrix[j, i] = val  # Symmetric
        
        return pd.DataFrame(matrix, index=factors, columns=factors)
    
    # =========================================================================
    # Query Methods
    # =========================================================================
    
    def list_analyses(
        self,
        asset_type: AssetType,
        factor_type: FactorType,
        limit: int = 20
    ) -> pd.DataFrame:
        """List recent orthogonality analyses."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        sql = f"""
            SELECT 
                analysis_id,
                analysis_datetime,
                total_factors,
                effective_n_90,
                n_clusters,
                n_selected,
                selection_ratio
            FROM {db_name}.B1_orthogonality_analysis
            ORDER BY analysis_datetime DESC
            LIMIT {limit}
        """
        
        return db.fetch(sql)
    
    def get_analysis_summary(
        self,
        analysis_id: str,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> Optional[Dict[str, Any]]:
        """Get analysis summary by ID."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        sql = f"""
            SELECT *
            FROM {db_name}.B1_orthogonality_analysis
            WHERE analysis_id = '{analysis_id}'
            LIMIT 1
        """
        
        result = db.fetch(sql)
        if len(result) == 0:
            return None
        
        return result.iloc[0].to_dict()
    
    def get_selected_factors(
        self,
        analysis_id: str,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> List[str]:
        """Get list of selected factors from an analysis."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        sql = f"""
            SELECT factor_id
            FROM {db_name}.B6_factor_selection
            WHERE analysis_id = '{analysis_id}' AND is_selected = 1
            ORDER BY selection_order
        """
        
        result = db.fetch(sql)
        return result['factor_id'].tolist()
    
    def get_cluster_details(
        self,
        analysis_id: str,
        asset_type: AssetType,
        factor_type: FactorType
    ) -> pd.DataFrame:
        """Get factor cluster assignments and details."""
        db, db_name = self._get_db(asset_type, factor_type)
        
        sql = f"""
            SELECT *
            FROM {db_name}.B3_factor_clusters
            WHERE analysis_id = '{analysis_id}'
            ORDER BY cluster_id, centrality_score DESC
        """
        
        return db.fetch(sql)
