import pandas as pd
import numpy as np
from numba import njit, prange
from scipy import stats, optimize, linalg
from scipy.stats import norm
from sklearn.decomposition import PCA
from sklearn.covariance import LedoitWolf
import warnings
from collections import defaultdict, deque
import logging
from scipy.optimize import minimize
from Backtester_Tushar.Risk.risk_limits import RiskLimits
from Backtester_Tushar.Portfolio_Management_and_Tracker.utils import *
from Backtester_Tushar.Strategy.signal_processing import SignalProcessor
from Backtester_Tushar.Execution.t_cost import MarketImpactModel
from Backtester_Tushar.Risk.risk_manager import RiskManager
from Backtester_Tushar.Portfolio_Management_and_Tracker.analyzer import PerformanceAnalyzer

warnings.filterwarnings('ignore')


class PortfolioManager:
    """Institutional-grade portfolio management system"""

    def __init__(self,
                 initial_capital=10000000,
                 n_long_positions=20,
                 k_short_positions=20,
                 long_capital_pct=0.6,
                 short_capital_pct=0.4,
                 rebalance_frequency=1,
                 risk_limits=None,
                 deployable_capital_pct=1.0,
                 transaction_cost_bps=5.0,
                 slippage_factor=0.1):

        # Core parameters
        self.initial_capital = initial_capital
        self.current_capital = initial_capital
        self.n_long_positions = n_long_positions
        self.k_short_positions = k_short_positions
        self.long_capital_pct = long_capital_pct
        self.short_capital_pct = short_capital_pct
        self.rebalance_frequency = rebalance_frequency
        self.risk_limits = risk_limits or RiskLimits()

        # Initialize components
        self.risk_manager = RiskManager(self.risk_limits)
        self.signal_processor = SignalProcessor()
        self.market_impact_model = MarketImpactModel()
        self.performance_analyzer = PerformanceAnalyzer()

        self.deployable_capital_pct = deployable_capital_pct

        # Store dynamic allocation parameters
        self.current_long_pct = long_capital_pct
        self.current_short_pct = short_capital_pct
        self.current_deployable_pct = deployable_capital_pct

        # Update risk limits to enforce no leverage
        self.risk_limits.max_leverage = 1.0

        # Initialize tracking variables
        self.positions = {}
        self.trade_log = []
        self.performance_log = []
        self.risk_log = []
        self.returns_covariance = None
        self.ticker_order = []
        self.betas = {}
        self.historical_returns = []
        self.portfolio_values = []
        self.returns = []
        self.peak_value = initial_capital
        self.current_drawdown = 0.0

        # Fixed transaction costs and slippage
        self.transaction_cost_bps = transaction_cost_bps / 10000
        self.slippage_factor = slippage_factor

        self.trade_id = 1
        self.last_rebalance_date = None

        # Initialize missing attributes that are referenced in methods
        self.max_position_change_pct = None
        self.position_change_constraints = {}
        self.existing_position_discount = 0.5
        self.transition_cost_penalty_multiplier = 2.0

        self.setup_logging()

    def setup_logging(self):
        """Setup comprehensive logging system"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('portfolio_management.log'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger('PortfolioManager')

    def update_allocation_parameters(self, long_pct=None, short_pct=None, deployable_pct=None):
        """Update allocation parameters for next rebalancing."""
        if long_pct is not None:
            self.current_long_pct = long_pct
        if short_pct is not None:
            self.current_short_pct = short_pct
        if deployable_pct is not None:
            self.current_deployable_pct = deployable_pct

        # Ensure allocations are valid
        total_allocation = self.current_long_pct + self.current_short_pct
        if total_allocation > self.current_deployable_pct:
            self.logger.warning(
                f"Total allocation ({total_allocation:.2%}) exceeds deployable capital ({self.current_deployable_pct:.2%})")
            # Auto-adjust to fit within deployable capital
            scale_factor = self.current_deployable_pct / total_allocation
            self.current_long_pct *= scale_factor
            self.current_short_pct *= scale_factor
            self.logger.info(
                f"Adjusted allocations - Long: {self.current_long_pct:.2%}, Short: {self.current_short_pct:.2%}")

    def validate_no_leverage(self, positions, portfolio_value):
        """Ensure gross exposure never exceeds portfolio value"""
        gross_exposure = sum(abs(pos.get('market_value', 0)) for pos in positions.values())
        return gross_exposure <= portfolio_value * (1 + 1e-6)

    def calculate_available_deployment_capital(self, portfolio_state):
        """Calculate actual cash available for new deployments"""
        total_portfolio_value = portfolio_state['total_portfolio_value']
        available_cash = portfolio_state['cash']
        current_position_value = portfolio_state['total_position_value']

        # Maximum deployment based on target allocation
        max_target_deployment = total_portfolio_value * self.current_deployable_pct

        # Already deployed capital
        current_deployment = abs(current_position_value)

        # Available for new deployments
        theoretical_available = max(0, max_target_deployment - current_deployment)
        actual_available = min(available_cash, theoretical_available)

        return {
            'total_portfolio_value': total_portfolio_value,
            'available_cash': available_cash,
            'current_deployment': current_deployment,
            'max_target_deployment': max_target_deployment,
            'theoretical_available': theoretical_available,
            'actual_available': actual_available,
            'deployment_ratio': current_deployment / total_portfolio_value if total_portfolio_value > 0 else 0
        }

    def _calculate_positions_for_side(self, portfolio_weights, available_capital, market_data, direction):
        """Calculate positions for one side (long or short) with precise capital allocation."""
        positions = {}

        if not portfolio_weights or available_capital <= 0:
            return positions

        # Normalize weights
        total_weight = sum(portfolio_weights.values())
        if total_weight <= 0:
            return positions

        remaining_capital = available_capital

        for ticker, weight in portfolio_weights.items():
            if remaining_capital <= 0:
                break

            ticker_data = market_data[market_data['ticker'] == ticker]
            if len(ticker_data) == 0:
                continue

            base_price = ticker_data.iloc[0]['Open']
            atr = ticker_data.iloc[0].get('ATR', 0.02)

            # Calculate effective price including slippage
            is_buy = (direction == 1)
            slippage_pct = self.slippage_factor * atr
            effective_price = base_price * (1 + slippage_pct if is_buy else 1 - slippage_pct)

            # Calculate target allocation
            normalized_weight = weight / total_weight
            target_capital_allocation = available_capital * normalized_weight

            # Calculate shares including transaction costs
            cost_multiplier = 1 + self.transaction_cost_bps
            affordable_shares = target_capital_allocation / (effective_price * cost_multiplier)
            final_shares = int(affordable_shares) * direction

            if abs(final_shares) >= 1:
                actual_trade_value = abs(final_shares) * effective_price
                actual_transaction_cost = actual_trade_value * self.transaction_cost_bps
                actual_total_cost = actual_trade_value + actual_transaction_cost

                positions[ticker] = {
                    'shares': final_shares,
                    'price': base_price,
                    'effective_price': effective_price,
                    'atr': atr,
                    'direction': direction,
                    'target_notional': target_capital_allocation,
                    'actual_cost': actual_total_cost
                }

                remaining_capital -= actual_total_cost

        # Log allocation
        total_allocated = sum(pos['actual_cost'] for pos in positions.values())
        utilization = total_allocated / available_capital if available_capital > 0 else 0
        side_name = "Long" if direction == 1 else "Short"
        self.logger.info(
            f"{side_name} allocation: ${total_allocated:,.0f} / ${available_capital:,.0f} ({utilization:.1%})")

        return positions

    def estimate_covariance_matrix(self, returns_data, method='ledoit_wolf'):
        """Estimate returns covariance matrix using advanced methods"""
        if returns_data is None or len(returns_data) < 20:
            return None

        returns_matrix = returns_data.values if hasattr(returns_data, 'values') else returns_data

        # Remove NaN rows
        valid_data = returns_matrix[~np.isnan(returns_matrix).all(axis=1)]
        if len(valid_data) < 10:
            return None

        try:
            if method == 'ledoit_wolf':
                lw = LedoitWolf().fit(valid_data)
                cov_matrix = lw.covariance_
            elif method == 'sample':
                cov_matrix = np.cov(valid_data.T)
            else:
                # Exponentially weighted covariance
                weights = np.exp(-0.01 * np.arange(len(valid_data))[::-1])
                weights = weights / np.sum(weights)
                weighted_data = valid_data * weights[:, np.newaxis]
                cov_matrix = np.cov(weighted_data.T)

            return cov_matrix

        except Exception as e:
            self.logger.warning(f"Covariance estimation failed: {e}")
            return None

    def select_optimal_portfolio(self, signals_df, returns_data=None):
        """Select optimal portfolio using advanced signal processing"""
        if len(signals_df) == 0:
            return {}, {}

        # Apply signal decay
        decayed_signals = self.signal_processor.decay_signals(signals_df)

        # Select portfolios
        long_portfolio = self._select_long_portfolio(decayed_signals, returns_data)
        short_portfolio = self._select_short_portfolio(decayed_signals, returns_data)

        return long_portfolio, short_portfolio

    def _select_long_portfolio(self, signals_df, returns_data):
        """Select optimal long portfolio"""
        if len(signals_df) == 0:
            return {}

        # Filter for positive signals
        long_signals = signals_df[signals_df['signal_descriptor'] > 0].copy()
        if len(long_signals) == 0:
            return {}

        long_signals_sorted = long_signals.nlargest(
            min(self.n_long_positions * 2, len(long_signals)), 'signal_descriptor'
        )

        scores = []
        tickers = []
        for _, row in long_signals_sorted.iterrows():
            base_score = row['signal_descriptor'] / max(row['ATR'], 0.001)
            market_impact = self.market_impact_model.calculate_market_impact(
                1000, row.get('Volume', 1000000), row['ATR']
            )
            adjusted_score = base_score * (1 - market_impact)
            scores.append(adjusted_score)
            tickers.append(row['ticker'])

        # Try optimization if possible
        if returns_data is not None and len(scores) > 1 and self.returns_covariance is not None:
            try:
                expected_returns = np.array(scores)
                expected_returns = expected_returns / np.sum(np.abs(expected_returns))

                ticker_indices = [self.ticker_order.index(t) for t in tickers if t in self.ticker_order]
                if len(ticker_indices) <= self.returns_covariance.shape[0]:
                    subset_cov = self.returns_covariance[np.ix_(ticker_indices, ticker_indices)]

                    # Build sector groups
                    sector_groups = defaultdict(list)
                    for i, ticker in enumerate(tickers):
                        sector_data = signals_df[signals_df['ticker'] == ticker]
                        sector = sector_data['sector'].iloc[0] if len(
                            sector_data) > 0 and 'sector' in sector_data.columns else 'Unknown'
                        sector_groups[sector].append(i)

                    betas = np.array([self.betas.get(t, 1.0) for t in tickers])
                    optimal_weights = self.risk_manager.optimize_with_constraints(
                        expected_returns, subset_cov, sector_groups, betas
                    )

                    portfolio = {}
                    for i, ticker in enumerate(tickers[:len(optimal_weights)]):
                        if abs(optimal_weights[i]) > 0.01:
                            portfolio[ticker] = optimal_weights[i]
                    return portfolio

            except Exception as e:
                self.logger.warning(f"Long optimization failed, using softmax: {e}")

        # Fallback to softmax
        if len(scores) > 0:
            weights = fast_softmax(np.array(scores))
            return dict(zip(tickers[:self.n_long_positions], weights[:self.n_long_positions]))
        return {}

    def _select_short_portfolio(self, signals_df, returns_data):
        """Select optimal short portfolio"""
        if len(signals_df) == 0:
            return {}

        # Filter for negative signals
        short_signals = signals_df[signals_df['signal_descriptor'] < 0].copy()
        if len(short_signals) == 0:
            return {}

        short_signals_sorted = short_signals.nsmallest(
            min(self.k_short_positions * 2, len(short_signals)), 'signal_descriptor'
        )

        scores = []
        tickers = []
        for _, row in short_signals_sorted.iterrows():
            base_score = abs(row['signal_descriptor']) / max(row['ATR'], 0.001)
            market_impact = self.market_impact_model.calculate_market_impact(
                1000, row.get('Volume', 1000000), row['ATR']
            )
            # Dynamic borrow cost
            borrow_cost = (row['ATR'] / 0.02) * (0.02 / 252)
            adjusted_score = base_score * (1 - market_impact - borrow_cost)
            scores.append(adjusted_score)
            tickers.append(row['ticker'])

        # Try optimization if possible
        if returns_data is not None and len(scores) > 1 and self.returns_covariance is not None:
            try:
                expected_returns = np.array(scores)
                expected_returns = expected_returns / np.sum(np.abs(expected_returns))

                ticker_indices = [self.ticker_order.index(t) for t in tickers if t in self.ticker_order]
                if len(ticker_indices) <= self.returns_covariance.shape[0]:
                    subset_cov = self.returns_covariance[np.ix_(ticker_indices, ticker_indices)]

                    sector_groups = defaultdict(list)
                    for i, ticker in enumerate(tickers):
                        sector_data = signals_df[signals_df['ticker'] == ticker]
                        sector = sector_data['sector'].iloc[0] if len(
                            sector_data) > 0 and 'sector' in sector_data.columns else 'Unknown'
                        sector_groups[sector].append(i)

                    betas = np.array([self.betas.get(t, 1.0) for t in tickers])
                    optimal_weights = self.risk_manager.optimize_with_constraints(
                        expected_returns, subset_cov, sector_groups, betas
                    )

                    portfolio = {}
                    for i, ticker in enumerate(tickers[:len(optimal_weights)]):
                        if abs(optimal_weights[i]) > 0.01:
                            portfolio[ticker] = optimal_weights[i]
                    return portfolio

            except Exception as e:
                self.logger.warning(f"Short optimization failed, using softmax: {e}")

        # Fallback to softmax
        if len(scores) > 0:
            weights = fast_softmax(np.array(scores))
            return dict(zip(tickers[:self.k_short_positions], weights[:self.k_short_positions]))
        return {}

    def calculate_portfolio_value_and_cash(self, market_data):
        """Calculate current portfolio value and available cash"""
        cash = self.current_capital
        total_position_value = 0.0
        position_details = {}

        for ticker, pos in self.positions.items():
            ticker_data = market_data[market_data['ticker'] == ticker]
            if len(ticker_data) > 0:
                current_price = ticker_data.iloc[0]['Close']
                market_value = pos['shares'] * current_price
                total_position_value += market_value

                position_details[ticker] = {
                    'shares': pos['shares'],
                    'current_price': current_price,
                    'market_value': market_value,
                    'direction': pos['direction']
                }

        total_portfolio_value = cash + total_position_value

        return {
            'total_portfolio_value': total_portfolio_value,
            'cash': cash,
            'total_position_value': total_position_value,
            'position_details': position_details
        }

    def calculate_target_positions_with_total_capital(self, long_portfolio, short_portfolio,
                                                      total_deployable_capital, market_data):
        """Calculate target positions using TOTAL deployable capital"""
        target_positions = {}

        # Calculate allocation for each side
        total_allocation_weight = self.current_long_pct + self.current_short_pct
        if total_allocation_weight <= 0:
            return target_positions

        long_capital = total_deployable_capital * (self.current_long_pct / total_allocation_weight)
        short_capital = total_deployable_capital * (self.current_short_pct / total_allocation_weight)

        self.logger.info(f"Capital allocation - Long: ${long_capital:,.0f}, Short: ${short_capital:,.0f}")

        # Calculate positions for each side
        if long_portfolio and long_capital > 0:
            long_positions = self._calculate_positions_for_side(
                long_portfolio, long_capital, market_data, direction=1
            )
            target_positions.update(long_positions)

        if short_portfolio and short_capital > 0:
            short_positions = self._calculate_positions_for_side(
                short_portfolio, short_capital, market_data, direction=-1
            )
            target_positions.update(short_positions)

        return target_positions

    def apply_position_change_constraints(self, target_positions, current_positions):
        """Apply position change constraints (placeholder for future implementation)"""
        if not self.max_position_change_pct and not self.position_change_constraints:
            return target_positions

        constrained_positions = target_positions.copy()

        # Global position change constraint
        if self.max_position_change_pct:
            for ticker, target_pos in target_positions.items():
                current_pos = current_positions.get(ticker, {})
                current_shares = current_pos.get('shares', 0)
                target_shares = target_pos.get('shares', 0)  # Fixed: use 'shares' not 'target_shares'

                if current_shares != 0:
                    change_pct = abs(target_shares - current_shares) / abs(current_shares)

                    if change_pct > self.max_position_change_pct:
                        max_change = abs(current_shares) * self.max_position_change_pct
                        if target_shares > current_shares:
                            capped_shares = current_shares + max_change
                        else:
                            capped_shares = current_shares - max_change

                        self.logger.info(f"Position change constraint applied to {ticker}: "
                                         f"{target_shares:.0f} -> {capped_shares:.0f} shares")

                        constrained_positions[ticker]['shares'] = int(capped_shares)  # Fixed key

        return constrained_positions

    def calculate_minimal_trades(self, target_positions, market_data):
        """Calculate minimal trades needed to reach target portfolio"""
        trades_needed = []

        # Get current and target positions
        current_tickers = set(self.positions.keys())
        target_tickers = set(target_positions.keys())
        all_tickers = current_tickers | target_tickers

        for ticker in all_tickers:
            current_shares = self.positions.get(ticker, {}).get('shares', 0)
            target_shares = target_positions.get(ticker, {}).get('shares', 0)  # Fixed key

            shares_delta = target_shares - current_shares

            if abs(shares_delta) < 1:
                continue

            ticker_data = market_data[market_data['ticker'] == ticker]
            if ticker_data.empty:
                continue

            price = ticker_data.iloc[0]['Close']
            atr = ticker_data.iloc[0].get('ATR', 0.02)

            # Apply slippage
            is_buy = shares_delta > 0
            slippage_pct = self.slippage_factor * atr
            effective_price = price * (1 + slippage_pct if is_buy else 1 - slippage_pct)

            # Calculate net investment
            trade_value = abs(shares_delta * effective_price)
            transaction_cost = trade_value * self.transaction_cost_bps

            if is_buy:
                net_investment = trade_value + transaction_cost
            else:
                net_investment = -(trade_value - transaction_cost)

            trades_needed.append({
                'ticker': ticker,
                'current_shares': current_shares,
                'target_shares': target_shares,
                'shares_delta': shares_delta,
                'price': price,
                'effective_price': effective_price,
                'atr': atr,
                'net_investment': net_investment,
                'is_buy': is_buy
            })

        # Sort by net investment (cash-generating trades first)
        trades_needed.sort(key=lambda x: x['net_investment'])

        self.logger.info(f"Calculated {len(trades_needed)} minimal trades")
        return trades_needed

    def scale_trades_to_available_capital(self, trades_needed, available_cash):
        """Scale down capital-requiring trades if insufficient cash available"""
        cash_generating = [t for t in trades_needed if t['net_investment'] < 0]
        cash_requiring = [t for t in trades_needed if t['net_investment'] > 0]

        # Calculate available capital
        total_cash_generated = sum(abs(t['net_investment']) for t in cash_generating)
        total_available = available_cash + total_cash_generated
        total_capital_required = sum(t['net_investment'] for t in cash_requiring)

        if total_capital_required > total_available:
            scale_factor = total_available / total_capital_required
            self.logger.warning(f"Scaling down trades by {scale_factor:.3f} due to capital constraints")

            for trade in cash_requiring:
                original_shares = trade['shares_delta']
                scaled_shares = int(original_shares * scale_factor)

                if abs(scaled_shares) >= 1:
                    trade['shares_delta'] = scaled_shares
                    # Recalculate net investment
                    trade_value = abs(scaled_shares * trade['effective_price'])
                    transaction_cost = trade_value * self.transaction_cost_bps
                    trade['net_investment'] = trade_value + transaction_cost
                else:
                    trade['shares_delta'] = 0
                    trade['net_investment'] = 0

        return cash_generating + cash_requiring

    def execute_smart_rebalancing(self, long_portfolio, short_portfolio, current_date, signal_data, market_data):
        """Execute portfolio rebalancing with proper capital calculation"""
        self.logger.info(f"Starting rebalancing for {current_date}")

        trades_executed = []
        total_transaction_costs = 0.0

        # Calculate current portfolio state
        portfolio_state = self.calculate_portfolio_value_and_cash(market_data)
        total_portfolio_value = portfolio_state['total_portfolio_value']
        total_deployable_capital = total_portfolio_value * self.current_deployable_pct

        self.logger.info(f"Current portfolio value: ${total_portfolio_value:,.2f}")
        self.logger.info(f"Total deployable capital: ${total_deployable_capital:,.2f}")

        # Calculate target positions
        target_positions = self.calculate_target_positions_with_total_capital(
            long_portfolio, short_portfolio, total_deployable_capital, market_data
        )

        # Apply constraints if enabled
        if self.max_position_change_pct or self.position_change_constraints:
            target_positions = self.apply_position_change_constraints(target_positions, self.positions)

        self.logger.info(f"Calculated {len(target_positions)} target positions")

        # Calculate minimal trades needed
        trades_needed = self.calculate_minimal_trades(target_positions, market_data)

        # Scale trades if needed
        available_cash = portfolio_state['cash']
        total_trade_capital_required = sum(t['net_investment'] for t in trades_needed if t['net_investment'] > 0)

        if total_trade_capital_required > available_cash:
            self.logger.warning(
                f"Insufficient cash. Required: ${total_trade_capital_required:,.0f}, Available: ${available_cash:,.0f}")
            trades_needed = self.scale_trades_to_available_capital(trades_needed, available_cash)

        # Execute trades
        working_cash = available_cash

        for trade in trades_needed:
            if abs(trade['shares_delta']) < 1:
                continue

            ticker = trade['ticker']
            shares_delta = trade['shares_delta']
            execution_price = trade['price']
            effective_price = trade['effective_price']
            atr = trade['atr']

            # Calculate costs
            trade_value = abs(shares_delta * effective_price)
            transaction_cost = trade_value * self.transaction_cost_bps
            cash_flow = -shares_delta * effective_price
            net_cash_impact = cash_flow - transaction_cost

            # Update working cash
            working_cash += net_cash_impact
            total_transaction_costs += transaction_cost

            # Update positions
            if ticker in self.positions:
                old_pos = self.positions[ticker]
                new_shares = old_pos['shares'] + shares_delta

                if abs(new_shares) > 0.01:
                    # Update existing position
                    old_cost_basis = old_pos['shares'] * old_pos['avg_price']
                    new_cost_basis = shares_delta * effective_price
                    total_cost_basis = old_cost_basis + new_cost_basis
                    new_avg_price = total_cost_basis / new_shares if new_shares != 0 else effective_price

                    self.positions[ticker].update({
                        'shares': new_shares,
                        'avg_price': new_avg_price,
                        'volatility': atr
                    })
                    action = 'ADJUST'
                else:
                    # Close position
                    del self.positions[ticker]
                    action = 'CLOSE'
            else:
                # Open new position
                direction = 1 if shares_delta > 0 else -1
                self.positions[ticker] = {
                    'shares': shares_delta,
                    'avg_price': effective_price,
                    'direction': direction,
                    'entry_date': current_date,
                    'volatility': atr
                }
                action = 'OPEN'

            # Record trade
            trades_executed.append({
                'trade_id': self.trade_id,
                'date': current_date,
                'ticker': ticker,
                'action': action,
                'shares': shares_delta,
                'execution_price': effective_price,
                'direction': 1 if shares_delta > 0 else -1,
                'transaction_cost': transaction_cost,
                'slippage': effective_price - execution_price,
                'cash_flow': net_cash_impact
            })

            self.trade_id += 1

        # Update cash
        self.current_capital = working_cash

        # Final validation
        final_portfolio_state = self.calculate_portfolio_value_and_cash(market_data)
        leverage_ratio = abs(final_portfolio_state['total_position_value']) / final_portfolio_state[
            'total_portfolio_value']
        leverage_compliant = leverage_ratio <= self.risk_limits.max_leverage

        self.logger.info(f"Rebalancing complete:")
        self.logger.info(f"  - Executed {len(trades_executed)} trades")
        self.logger.info(f"  - Transaction costs: ${total_transaction_costs:,.2f}")
        self.logger.info(f"  - Final leverage: {leverage_ratio:.3f}")
        self.logger.info(f"  - Leverage compliant: {leverage_compliant}")

        self.trade_log.extend(trades_executed)
        return total_transaction_costs, len(trades_executed)

    def calculate_comprehensive_portfolio_metrics(self, market_data, current_date):
        """Calculate comprehensive portfolio metrics"""
        # Get current portfolio state
        portfolio_state = self.calculate_portfolio_value_and_cash(market_data)

        cash = portfolio_state['cash']
        total_position_value = portfolio_state['total_position_value']
        portfolio_value = portfolio_state['total_portfolio_value']
        position_details = portfolio_state['position_details']

        # Calculate position metrics
        total_unrealized_pnl = 0.0
        total_borrow_cost = 0.0
        long_exposure = 0.0
        short_exposure = 0.0

        for ticker, pos_detail in position_details.items():
            pos = self.positions[ticker]

            cost_basis = pos['shares'] * pos['avg_price']
            market_value = pos_detail['market_value']

            if pos['direction'] == 1:
                unrealized_pnl = market_value - cost_basis
                long_exposure += abs(market_value)
            else:
                unrealized_pnl = cost_basis - market_value
                short_exposure += abs(market_value)

                # Calculate borrow cost for short positions
                atr = market_data[market_data['ticker'] == ticker].iloc[0].get('ATR', 0.02)
                borrow_cost = (atr / 0.02) * (0.02 / 252) * abs(market_value)
                total_borrow_cost += borrow_cost

            total_unrealized_pnl += unrealized_pnl

            position_details[ticker].update({
                'avg_price': pos['avg_price'],
                'cost_basis': cost_basis,
                'unrealized_pnl': unrealized_pnl,
                'weight': market_value / portfolio_value if portfolio_value > 0 else 0,
                'days_held': (pd.to_datetime(current_date) - pd.to_datetime(pos['entry_date'])).days
            })

        # Calculate returns
        if len(self.portfolio_values) > 0:
            daily_return = (portfolio_value - self.portfolio_values[-1]) / self.portfolio_values[-1]
        else:
            daily_return = 0.0

        self.returns.append(daily_return)
        self.historical_returns.append(daily_return)

        # Update drawdown tracking
        if portfolio_value > self.peak_value:
            self.peak_value = portfolio_value
        current_drawdown = (self.peak_value - portfolio_value) / self.peak_value
        self.current_drawdown = current_drawdown

        # Calculate exposure metrics
        gross_exposure = long_exposure + short_exposure
        net_exposure = long_exposure - short_exposure
        deployed_capital = gross_exposure
        deployment_ratio = deployed_capital / portfolio_value if portfolio_value > 0 else 0
        cash_ratio = cash / portfolio_value if portfolio_value > 0 else 1.0
        leverage_ratio = gross_exposure / portfolio_value if portfolio_value > 0 else 0

        # Validate leverage compliance
        position_values = {ticker: {'market_value': pos['market_value']}
                           for ticker, pos in position_details.items()}
        leverage_compliant = self.validate_no_leverage(position_values, portfolio_value)

        # Additional compliance checks
        deployment_compliant = deployment_ratio <= self.current_deployable_pct * 1.01
        cash_positive = cash >= 0
        drawdown_compliant = current_drawdown <= self.risk_limits.max_drawdown

        # Calculate sector exposures and beta
        sector_exposures = self._calculate_sector_exposures(position_details, market_data)
        portfolio_beta = self._calculate_portfolio_beta(position_details, market_data)

        # Calculate VaR
        portfolio_var = 0.0
        if self.returns_covariance is not None:
            var_positions = {ticker: {'market_value': pos['market_value']}
                             for ticker, pos in position_details.items()}
            portfolio_var = self.risk_manager.calculate_portfolio_var(var_positions, self.returns_covariance)

        # Performance metrics
        performance_metrics = {}
        if len(self.returns) > 20:
            performance_metrics = self.performance_analyzer.calculate_performance_metrics(
                self.returns[-252:] if len(self.returns) >= 252 else self.returns
            )

        # Compile comprehensive metrics
        comprehensive_metrics = {
            # Basic portfolio metrics
            'date': current_date,
            'portfolio_value': portfolio_value,
            'cash': cash,
            'total_position_value': total_position_value,
            'total_unrealized_pnl': total_unrealized_pnl,
            'daily_return': daily_return,
            'current_drawdown': current_drawdown,

            # Position metrics
            'num_positions': len(self.positions),
            'num_long_positions': len([p for p in self.positions.values() if p['direction'] == 1]),
            'num_short_positions': len([p for p in self.positions.values() if p['direction'] == -1]),

            # Exposure metrics
            'long_exposure': long_exposure,
            'short_exposure': short_exposure,
            'gross_exposure': gross_exposure,
            'net_exposure': net_exposure,
            'deployed_capital': deployed_capital,

            # Ratios and compliance
            'deployment_ratio': deployment_ratio,
            'cash_ratio': cash_ratio,
            'leverage_ratio': leverage_ratio,
            'leverage': leverage_ratio,  # Add this alias for backwards compatibility
            'target_deployment_ratio': self.current_deployable_pct,
            'current_long_pct': self.current_long_pct,
            'current_short_pct': self.current_short_pct,

            # Risk metrics
            'portfolio_beta': portfolio_beta,
            'portfolio_var_95': portfolio_var,
            'total_borrow_cost': total_borrow_cost,
            'sector_exposures': sector_exposures,

            # Compliance flags
            'leverage_compliant': leverage_compliant,
            'deployment_compliant': deployment_compliant,
            'drawdown_compliant': drawdown_compliant,
            'cash_positive': cash_positive,
            'overall_compliant': all([leverage_compliant, deployment_compliant, drawdown_compliant, cash_positive]),

            # Position details
            'position_details': position_details
        }

        # Add performance metrics
        comprehensive_metrics.update(performance_metrics)

        # Store metrics
        self.portfolio_values.append(portfolio_value)
        self.performance_log.append(comprehensive_metrics)

        # Apply daily costs to cash only
        self.current_capital = cash - total_borrow_cost

        # Log compliance violations
        if not leverage_compliant:
            self.logger.error(
                f"LEVERAGE VIOLATION: Gross exposure ${gross_exposure:,.2f} > Portfolio value ${portfolio_value:,.2f}")
        if not cash_positive:
            self.logger.error(f"NEGATIVE CASH: ${cash:,.2f}")
        if not deployment_compliant:
            self.logger.warning(f"Deployment breach: {deployment_ratio:.3f} > {self.current_deployable_pct:.3f}")

        return comprehensive_metrics

    def _calculate_portfolio_beta(self, position_details, market_data):
        """Calculate portfolio beta using betas or ATR proxy"""
        total_beta_exposure = 0.0
        total_weight = 0.0

        for ticker, pos in position_details.items():
            beta = self.betas.get(ticker, None)
            if beta is None:
                ticker_data = market_data[market_data['ticker'] == ticker]
                if len(ticker_data) > 0:
                    ticker_atr = ticker_data.iloc[0].get('ATR', 0.02)
                    avg_atr = market_data['ATR'].mean() if 'ATR' in market_data.columns else 0.02
                    beta = ticker_atr / avg_atr if avg_atr > 0 else 1.0
                else:
                    beta = 1.0

            weight = abs(pos.get('weight', 0))
            direction = 1 if pos.get('market_value', 0) > 0 else -1
            total_beta_exposure += beta * weight * direction
            total_weight += weight

        return total_beta_exposure / total_weight if total_weight > 0 else 0.0

    def _calculate_sector_exposures(self, position_details, market_data):
        """Calculate sector exposures"""
        sector_exposures = defaultdict(float)

        for ticker, pos in position_details.items():
            ticker_data = market_data[market_data['ticker'] == ticker]
            if not ticker_data.empty and 'sector' in ticker_data.columns:
                sector = ticker_data['sector'].iloc[0]
            else:
                sector = 'Unknown'
            sector_exposures[sector] += pos.get('weight', 0)

        return dict(sector_exposures)

    def run_institutional_backtest(self, master_df, allocation_schedule=None):
        """Run comprehensive institutional-grade backtest"""
        self.logger.info("=" * 60)
        self.logger.info("STARTING INSTITUTIONAL PORTFOLIO BACKTEST")
        self.logger.info("=" * 60)
        self.logger.info(f"Initial capital: ${self.initial_capital:,.2f}")
        self.logger.info(f"Initial allocation - Long: {self.current_long_pct:.1%}, Short: {self.current_short_pct:.1%}")

        # Data preprocessing
        self.logger.info("Preprocessing data...")
        master_df = self._preprocess_data(master_df)

        # Initialize risk models
        self.logger.info("Initializing risk models...")
        self._initialize_risk_models(master_df)

        # Backtest execution
        unique_dates = sorted(master_df['date'].unique())
        results = []
        allocation_changes = 0

        self.logger.info(f"Processing {len(unique_dates)} trading days...")

        for i, current_date in enumerate(unique_dates):
            try:
                # Check for allocation parameter updates
                if allocation_schedule and current_date in allocation_schedule:
                    old_params = (self.current_long_pct, self.current_short_pct, self.current_deployable_pct)

                    params = allocation_schedule[current_date]
                    self.update_allocation_parameters(
                        long_pct=params.get('long_pct'),
                        short_pct=params.get('short_pct'),
                        deployable_pct=params.get('deployable_pct')
                    )

                    new_params = (self.current_long_pct, self.current_short_pct, self.current_deployable_pct)

                    if old_params != new_params:
                        allocation_changes += 1
                        self.logger.info(f"ALLOCATION UPDATE on {current_date}:")
                        self.logger.info(f"  Long: {old_params[0]:.1%} → {new_params[0]:.1%}")
                        self.logger.info(f"  Short: {old_params[1]:.1%} → {new_params[1]:.1%}")

                # Get current day data
                df_day = master_df[master_df['date'] == current_date].copy()

                # Get next day data for execution
                next_day_data = None
                execution_date = current_date

                if i < len(unique_dates) - 1:
                    next_date = unique_dates[i + 1]
                    next_day_data = master_df[master_df['date'] == next_date].copy()
                    execution_date = next_date

                # Handle signal conflicts
                df_day = self._detect_and_handle_conflicts(df_day)

                # Check rebalancing
                should_rebalance = self._should_rebalance(current_date)

                transaction_costs = 0.0
                trades_count = 0

                # Execute rebalancing
                if should_rebalance and len(df_day) > 0 and next_day_data is not None:
                    returns_window = self._get_returns_window(master_df, current_date, lookback=60)
                    long_portfolio, short_portfolio = self.select_optimal_portfolio(df_day, returns_window)

                    if len(long_portfolio) > 0 or len(short_portfolio) > 0:
                        transaction_costs, trades_count = self.execute_smart_rebalancing(
                            long_portfolio, short_portfolio, current_date, df_day, next_day_data
                        )
                        self.last_rebalance_date = current_date

                    daily_metrics = self.calculate_comprehensive_portfolio_metrics(next_day_data, execution_date)
                else:
                    daily_metrics = self.calculate_comprehensive_portfolio_metrics(df_day, current_date)

                # Add metadata
                daily_metrics.update({
                    'rebalanced': should_rebalance,
                    'transaction_costs': transaction_costs,
                    'trades_executed': trades_count,
                    'next_day_execution': next_day_data is not None,
                    'execution_date': execution_date,
                    'allocation_changed': allocation_schedule and current_date in allocation_schedule
                })

                results.append(daily_metrics)
                self._monitor_risk_limits(daily_metrics)

                # Progress reporting
                if i % 50 == 0 or i == len(unique_dates) - 1:
                    progress = (i + 1) / len(unique_dates) * 100
                    self.logger.info(
                        f"Progress: {progress:.1f}% - Date: {current_date} - Portfolio: ${daily_metrics['portfolio_value']:,.2f}")

            except Exception as e:
                self.logger.error(f"Error processing date {current_date}: {str(e)}")
                continue

        self.logger.info("BACKTEST COMPLETED")

        # Generate results
        results_df = pd.DataFrame(results)
        final_analytics = self._generate_final_analytics(results_df)
        risk_report = self._generate_risk_report(results_df)
        compliance_report = self._generate_compliance_report(results_df)
        allocation_report = self._generate_allocation_report(results_df, allocation_changes)

        # Log summary
        if len(results_df) > 0:
            final_value = results_df['portfolio_value'].iloc[-1]
            total_return = (final_value - self.initial_capital) / self.initial_capital
            max_dd = results_df['current_drawdown'].max()

            self.logger.info(f"FINAL RESULTS:")
            self.logger.info(f"  Final Value: ${final_value:,.2f}")
            self.logger.info(f"  Total Return: {total_return:.2%}")
            self.logger.info(f"  Max Drawdown: {max_dd:.2%}")

        return {
            'daily_results': results_df,
            'final_analytics': final_analytics,
            'risk_report': risk_report,
            'compliance_report': compliance_report,
            'allocation_report': allocation_report,
            'trade_log': pd.DataFrame(self.trade_log),
            'position_history': self._generate_position_history(),
            'backtest_summary': {
                'total_days': len(unique_dates),
                'successful_days': len(results),
                'allocation_changes': allocation_changes,
                'rebalance_days': len([r for r in results if r.get('rebalanced', False)])
            }
        }

    # Helper methods for backtest
    def _preprocess_data(self, master_df):
        """Data preprocessing with validation"""
        required_columns = ['date', 'ticker', 'signal_descriptor', 'Close', 'ATR', 'Volume']
        missing_columns = [col for col in required_columns if col not in master_df.columns]

        if missing_columns:
            raise ValueError(f"Missing required columns: {missing_columns}")

        # Clean data
        master_df = master_df.dropna(subset=['Close', 'ATR', 'signal_descriptor'])

        # Handle outliers
        atr_99th = master_df['ATR'].quantile(0.99)
        master_df['ATR'] = master_df['ATR'].clip(upper=atr_99th)

        # Ensure proper data types
        master_df['date'] = pd.to_datetime(master_df['date'])
        master_df = master_df.sort_values(['date', 'ticker']).reset_index(drop=True)

        # Add returns
        master_df['returns'] = master_df.groupby('ticker')['Close'].pct_change()

        self.logger.info(f"Data preprocessing complete. {len(master_df)} records processed.")
        return master_df

    def _initialize_risk_models(self, master_df):
        """Initialize risk models and covariance matrices"""
        returns_pivot = master_df.pivot_table(
            index='date', columns='ticker', values='returns', fill_value=0
        )
        self.ticker_order = returns_pivot.columns.tolist()

        if len(returns_pivot) > 60:
            self.returns_covariance = self.estimate_covariance_matrix(returns_pivot.values[-252:])
            self.logger.info("Risk models initialized successfully")
        else:
            self.logger.warning("Insufficient data for risk model initialization")

    def _detect_and_handle_conflicts(self, df_day):
        """Handle conflicting signals"""
        ticker_groups = df_day.groupby('ticker')

        for ticker, group in ticker_groups:
            signals = group['signal_descriptor'].dropna()

            if len(signals) > 1:
                has_pos = (signals > 0).any()
                has_neg = (signals < 0).any()

                if has_pos and has_neg:
                    # Keep strongest absolute signal
                    strongest_idx = group['signal_descriptor'].abs().idxmax()
                    df_day.loc[
                        (df_day['ticker'] == ticker) & (df_day.index != strongest_idx), 'signal_descriptor'] = 0.0

        return df_day

    def _should_rebalance(self, current_date):
        """Determine if rebalancing is needed"""
        if self.last_rebalance_date is None:
            return True

        days_since_rebalance = (pd.to_datetime(current_date) - pd.to_datetime(self.last_rebalance_date)).days

        if days_since_rebalance >= self.rebalance_frequency:
            return True

        # Emergency rebalancing
        if self.current_drawdown > self.risk_limits.max_drawdown * 0.8:
            self.logger.warning("Emergency rebalancing triggered")
            return True

        return False

    def _get_returns_window(self, master_df, current_date, lookback=60):
        """Get historical returns window"""
        current_date_dt = pd.to_datetime(current_date)
        start_date = current_date_dt - pd.Timedelta(days=lookback)

        return master_df[(master_df['date'] >= start_date) & (master_df['date'] < current_date)]

    def _monitor_risk_limits(self, daily_metrics):
        """Monitor risk limits and generate alerts"""
        alerts = []

        if daily_metrics['current_drawdown'] > self.risk_limits.max_drawdown:
            alerts.append(f"CRITICAL: Drawdown limit breached - {daily_metrics['current_drawdown']:.2%}")

        if daily_metrics.get('leverage_ratio', 0) > self.risk_limits.max_leverage:
            alerts.append(f"WARNING: Leverage limit exceeded - {daily_metrics['leverage_ratio']:.2f}")

        for alert in alerts:
            self.logger.warning(alert)

        self.risk_log.append({
            'date': daily_metrics['date'],
            'alerts': alerts,
            'risk_metrics': {
                'drawdown': daily_metrics['current_drawdown'],
                'leverage': daily_metrics.get('leverage_ratio', 0),
                'var_95': daily_metrics.get('portfolio_var_95', 0),
                'beta': daily_metrics.get('portfolio_beta', 0)
            }
        })

    def _generate_final_analytics(self, results_df):
        """Generate final performance analytics"""
        if len(results_df) == 0:
            return {}

        initial_value = self.initial_capital
        final_value = results_df['portfolio_value'].iloc[-1]
        total_return = (final_value - initial_value) / initial_value

        returns = results_df['daily_return'].dropna()
        performance_metrics = {}
        if len(returns) > 0:
            performance_metrics = self.performance_analyzer.calculate_performance_metrics(returns.values)

        total_trades = len(self.trade_log)
        total_transaction_costs = sum(trade.get('transaction_cost', 0) for trade in self.trade_log)

        return {
            'performance': {
                'initial_capital': initial_value,
                'final_value': final_value,
                'total_return': total_return,
                **performance_metrics
            },
            'risk': {
                'max_drawdown': results_df['current_drawdown'].max(),
                'max_leverage': results_df.get('leverage_ratio', pd.Series([0])).max(),
                'avg_positions': results_df['num_positions'].mean()
            },
            'trading': {
                'total_trades': total_trades,
                'total_transaction_costs': total_transaction_costs,
                'turnover_rate': total_transaction_costs / initial_value
            }
        }

    def _generate_risk_report(self, results_df):
        """Generate risk analysis report"""
        return {
            'drawdown_analysis': self._analyze_drawdowns(results_df),
            'concentration_analysis': self._analyze_concentration_risk(results_df)
        }

    def _analyze_drawdowns(self, results_df):
        """Analyze drawdown characteristics"""
        if 'current_drawdown' not in results_df.columns:
            return {}

        drawdowns = results_df['current_drawdown'].values
        return {
            'max_drawdown': drawdowns.max(),
            'avg_drawdown': drawdowns.mean()
        }

    def _analyze_concentration_risk(self, results_df):
        """Analyze portfolio concentration"""
        return {'avg_num_positions': results_df['num_positions'].mean()}

    def _generate_compliance_report(self, results_df):
        """Generate compliance report"""
        if len(results_df) == 0:
            return {}

        leverage_violations = results_df[results_df.get('leverage_ratio', 0) > self.risk_limits.max_leverage]
        return {
            'leverage_compliance': {
                'compliant': len(leverage_violations) == 0,
                'violations': len(leverage_violations)
            }
        }

    def _generate_allocation_report(self, results_df, allocation_changes):
        """Generate allocation report"""
        return {
            'allocation_changes': allocation_changes,
            'avg_deployment': results_df.get('deployment_ratio', pd.Series([0])).mean()
        }

    def _generate_position_history(self):
        """Generate position history"""
        return {
            'current_positions': self.positions,
            'total_trades': len(self.trade_log)
        }