"""
Data Analysis Shell - Bidirectional Pattern Detection & Idea Generation
HEX_ID: 44415441 (DATA - Tools/Shell/Analysis/Primary)

Standalone version for cubesquare.

INPUTS:
    - Token dictionaries with hex_id, morton, basin, layer, topics
    - Beehive index JSON data
    - Token relationship data

OUTPUTS:
    - DataPattern: pattern_id, pattern_type, description, tokens, confidence, basin, layer
    - DataIdea: idea_id, idea_type, title, description, action, confidence, i_logic_state
    - Cluster structures: bucket, tokens, size, centroid_morton
    - Hierarchy structures: prefix, depth, children, count
    - Sequence structures: tokens, length, start_morton, end_morton
    - Ecosystem connectivity: nodes, edges, hubs, bridges

COMMANDS:
    --status      System status
    --demo        Demo with sample data
    --full        Full analysis with rich presentation
    --patterns    Show discovered patterns
    --ideas       Show generated ideas
    --ecosystem   Ecosystem analysis

I-LOGIC INTEGRATION:
    DIRECT (1)     : CORE category     - Identity preservation
    INVERSE (-1)   : RELATION category - Negation reflection
    SPIRAL (+i)    : ACTION category   - Expansion growth
    CONVERGENT (-i): DATA category     - Center-seeking consolidation
    ABSORBED (0)   : META category     - Neutral baseline
"""

import json
import hashlib
from pathlib import Path
from typing import Dict, List, Tuple, Optional, Any, Set
from dataclasses import dataclass, field, asdict
from datetime import datetime
from collections import defaultdict

# Import from cubesquare core
from .._core.constants import PHI, K5, K11, PRIME_131

# Import I-Logic
from ..i_logic.states import ILogicState

# Try numpy for vectorized operations
try:
    import numpy as np
    NUMPY_AVAILABLE = True
except ImportError:
    NUMPY_AVAILABLE = False


# Semantic categories by basin range
SEMANTIC_CATEGORIES = {
    'CORE': (0, 25),
    'ACTION': (26, 50),
    'DATA': (51, 75),
    'RELATION': (76, 100),
    'META': (101, 130),
}

# I-Logic category mapping
I_LOGIC_CATEGORY_MAP = {
    'CORE': ('DIRECT', ILogicState.DIRECT, 'Identity preservation'),
    'ACTION': ('SPIRAL', ILogicState.SPIRAL, 'Expansion growth'),
    'DATA': ('ANTI_SPIRAL', ILogicState.ANTI_SPIRAL, 'Consolidation'),
    'RELATION': ('INVERSE', ILogicState.INVERSE, 'Reflection'),
    'META': ('ABSORBED', ILogicState.ABSORBED, 'Neutral baseline'),
}


def get_i_logic_for_category(category: str) -> Tuple[str, complex, str]:
    """Map semantic category to I-Logic state."""
    return I_LOGIC_CATEGORY_MAP.get(category, ('ABSORBED', ILogicState.ABSORBED, 'Unknown'))


def batch_categorize_basins(basins: 'np.ndarray') -> 'np.ndarray':
    """
    Vectorized basin to category mapping - NO PYTHON LOOPS.
    """
    if not NUMPY_AVAILABLE:
        return None

    boundaries = np.array([0, 26, 51, 76, 101, 131])
    categories = np.array(['CORE', 'ACTION', 'DATA', 'RELATION', 'META'])
    indices = np.searchsorted(boundaries[1:], basins, side='right')
    return categories[np.clip(indices, 0, len(categories) - 1)]


def batch_compute_morton_buckets(mortons: 'np.ndarray', bucket_size: int = 10) -> 'np.ndarray':
    """
    Vectorized Morton bucket computation - NO PYTHON LOOPS.
    """
    if not NUMPY_AVAILABLE:
        return None
    return np.floor_divide(mortons, bucket_size)


@dataclass
class DataPattern:
    """Pattern discovered in data analysis."""
    pattern_id: str
    pattern_type: str
    description: str
    tokens: List[str]
    confidence: float
    basin: int
    layer: str
    semantic_category: str
    relationships: List[Tuple[str, str, str]] = field(default_factory=list)
    discovered_at: datetime = field(default_factory=datetime.now)


@dataclass
class DataIdea:
    """Idea generated from data analysis with I-Logic integration."""
    idea_id: str
    idea_type: str
    title: str
    description: str
    action: str
    source_patterns: List[str]
    confidence: float
    layer: str
    semantic_category: str
    implementation_hint: str = ""
    i_logic_state: str = ""
    i_logic_value: Any = 0
    generated_at: datetime = field(default_factory=datetime.now)

    def __post_init__(self):
        """Auto-assign I-Logic from semantic category."""
        if not self.i_logic_state:
            i_logic = get_i_logic_for_category(self.semantic_category)
            self.i_logic_state = i_logic[0]
            self.i_logic_value = ILogicState.symbol(i_logic[1])


@dataclass
class EcosystemNode:
    """Node in the ecosystem relationship graph."""
    hex_id: str
    node_type: str
    layer: str
    basin: int
    semantic_category: str
    connections: List[str] = field(default_factory=list)
    metadata: Dict[str, Any] = field(default_factory=dict)


class DataStructureDiscovery:
    """
    Discover data structures in surrounding tokens.
    """

    def __init__(self):
        self.discovered_structures: List[Dict] = []

    def discover_clusters(self, tokens: List[Dict], radius: int = 10) -> List[Dict]:
        """Discover token clusters based on Morton proximity."""
        clusters = []

        if NUMPY_AVAILABLE and len(tokens) > 0:
            mortons = np.array([t.get('morton', 0) for t in tokens], dtype=np.uint64)
            hex_ids = np.array([t.get('hex_id', '') for t in tokens])
            buckets = batch_compute_morton_buckets(mortons, radius)
            unique_buckets, bucket_counts = np.unique(buckets, return_counts=True)

            bucket_mask = bucket_counts >= 2
            for bucket, count in zip(unique_buckets[bucket_mask], bucket_counts[bucket_mask]):
                bucket_indices = np.where(buckets == bucket)[0]
                cluster_hex_ids = hex_ids[bucket_indices].tolist()
                clusters.append({
                    'type': 'cluster',
                    'bucket': int(bucket),
                    'tokens': cluster_hex_ids,
                    'size': int(count),
                    'centroid_morton': int(bucket * radius + radius // 2),
                })
        else:
            by_morton = defaultdict(list)
            for token in tokens:
                bucket = token.get('morton', 0) // radius
                by_morton[bucket].append(token)

            clusters = [
                {
                    'type': 'cluster',
                    'bucket': bucket,
                    'tokens': [t.get('hex_id', '') for t in cluster_tokens],
                    'size': len(cluster_tokens),
                    'centroid_morton': bucket * radius + radius // 2,
                }
                for bucket, cluster_tokens in by_morton.items()
                if len(cluster_tokens) >= 2
            ]

        self.discovered_structures.extend(clusters)
        return clusters

    def discover_hierarchies(self, tokens: List[Dict]) -> List[Dict]:
        """Discover hierarchical relationships based on hex prefix."""
        hierarchies = []
        by_prefix: Dict[str, List[Dict]] = defaultdict(list)

        for token in tokens:
            hex_id = token.get('hex_id', '')
            for prefix_len in [2, 4, 6]:
                if len(hex_id) >= prefix_len:
                    prefix = hex_id[:prefix_len]
                    by_prefix[prefix].append(token)

        for prefix, group in by_prefix.items():
            if len(group) >= 2 and len(prefix) < 8:
                file_paths = [t.get('file_path', '') for t in group[:5]]
                file_summary = ', '.join([
                    f.split('/')[-1] if '/' in f else f.split('\\')[-1] if '\\' in f else f
                    for f in file_paths if f
                ][:3])
                hierarchies.append({
                    'type': 'hierarchy',
                    'prefix': prefix,
                    'depth': len(prefix) // 2,
                    'children': [t.get('hex_id', '') for t in group],
                    'count': len(group),
                    'files': file_summary,
                    'layers': list(set(str(t.get('layer', 'L1')) for t in group[:10])),
                })

        self.discovered_structures.extend(hierarchies)
        return hierarchies

    def discover_sequences(self, tokens: List[Dict]) -> List[Dict]:
        """Discover sequential progressions in Morton codes."""
        sequences = []
        sorted_tokens = sorted(tokens, key=lambda t: t.get('morton', 0))

        current_seq = []
        for i, token in enumerate(sorted_tokens):
            if i == 0:
                current_seq = [token]
            else:
                prev_morton = sorted_tokens[i-1].get('morton', 0)
                curr_morton = token.get('morton', 0)
                if curr_morton - prev_morton <= 2:
                    current_seq.append(token)
                else:
                    if len(current_seq) >= 3:
                        sequences.append({
                            'type': 'sequence',
                            'tokens': [t.get('hex_id', '') for t in current_seq],
                            'length': len(current_seq),
                            'start_morton': current_seq[0].get('morton', 0),
                            'end_morton': current_seq[-1].get('morton', 0),
                        })
                    current_seq = [token]

        if len(current_seq) >= 3:
            sequences.append({
                'type': 'sequence',
                'tokens': [t.get('hex_id', '') for t in current_seq],
                'length': len(current_seq),
                'start_morton': current_seq[0].get('morton', 0),
                'end_morton': current_seq[-1].get('morton', 0),
            })

        self.discovered_structures.extend(sequences)
        return sequences


class EcosystemRelationshipAnalyzer:
    """Analyze relationships in the ecosystem."""

    def __init__(self):
        self.nodes: Dict[str, EcosystemNode] = {}
        self.edges: List[Tuple[str, str, str]] = []

    def add_node(self, hex_id: str, node_type: str, basin: int = 0,
                 layer: str = 'L1', metadata: Dict = None) -> EcosystemNode:
        """Add a node to the ecosystem graph."""
        category = self._get_category(basin)
        node = EcosystemNode(
            hex_id=hex_id,
            node_type=node_type,
            layer=layer,
            basin=basin,
            semantic_category=category,
            metadata=metadata or {}
        )
        self.nodes[hex_id] = node
        return node

    def add_edge(self, from_hex: str, to_hex: str, edge_type: str):
        """Add a relationship edge."""
        self.edges.append((from_hex, to_hex, edge_type))
        if from_hex in self.nodes:
            self.nodes[from_hex].connections.append(to_hex)

    def _get_category(self, basin: int) -> str:
        """Get semantic category from basin."""
        for category, (start, end) in SEMANTIC_CATEGORIES.items():
            if start <= basin <= end:
                return category
        return 'META'

    def analyze_connectivity(self) -> Dict:
        """Analyze graph connectivity statistics."""
        if not self.nodes:
            return {'error': 'No nodes in graph'}

        in_degree = defaultdict(int)
        out_degree = defaultdict(int)

        for from_hex, to_hex, _ in self.edges:
            out_degree[from_hex] += 1
            in_degree[to_hex] += 1

        hubs = sorted(
            [(h, out_degree[h] + in_degree[h]) for h in self.nodes],
            key=lambda x: x[1],
            reverse=True
        )[:10]

        category_count = defaultdict(int)
        for node in self.nodes.values():
            category_count[node.semantic_category] += 1

        return {
            'total_nodes': len(self.nodes),
            'total_edges': len(self.edges),
            'avg_degree': len(self.edges) * 2 / max(len(self.nodes), 1),
            'hubs': hubs,
            'category_distribution': dict(category_count),
        }

    def find_isolated_nodes(self) -> List[str]:
        """Find nodes with no connections."""
        connected = set()
        for from_hex, to_hex, _ in self.edges:
            connected.add(from_hex)
            connected.add(to_hex)
        return [h for h in self.nodes if h not in connected]

    def find_bridges(self) -> List[Tuple[str, str]]:
        """Find bridge edges between different categories."""
        bridges = []
        for from_hex, to_hex, edge_type in self.edges:
            from_node = self.nodes.get(from_hex)
            to_node = self.nodes.get(to_hex)
            if from_node and to_node:
                if from_node.semantic_category != to_node.semantic_category:
                    bridges.append((from_hex, to_hex))
        return bridges


class IdeaGenerator:
    """Generate ideas from patterns and analysis."""

    def __init__(self):
        self.ideas: List[DataIdea] = []
        self.idea_counter = 0

    def _generate_id(self) -> str:
        """Generate unique idea ID."""
        self.idea_counter += 1
        timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
        return f"IDEA-{timestamp}-{self.idea_counter:04d}"

    def generate_from_cluster(self, cluster: Dict) -> DataIdea:
        """Generate idea from cluster pattern."""
        idea = DataIdea(
            idea_id=self._generate_id(),
            idea_type='CONSOLIDATE',
            title=f"Consolidate {cluster['size']} related tokens",
            description=f"Cluster of {cluster['size']} tokens found at Morton bucket {cluster['bucket']}",
            action="Create unified concept or abstraction",
            source_patterns=[f"cluster:{cluster['bucket']}"],
            confidence=min(1.0, cluster['size'] / 10),
            layer='L2',
            semantic_category='CORE',
            implementation_hint="Consider creating a wrapper or composite pattern"
        )
        self.ideas.append(idea)
        return idea

    def generate_from_hierarchy(self, hierarchy: Dict) -> DataIdea:
        """Generate idea from hierarchy pattern."""
        files_hint = hierarchy.get('files', '')
        layers = hierarchy.get('layers', ['L1'])
        layers = [str(l) if not isinstance(l, str) else l for l in layers]
        layer_str = '/'.join(layers[:3])

        idea = DataIdea(
            idea_id=self._generate_id(),
            idea_type='SPECIALIZE' if hierarchy['depth'] > 2 else 'ELEVATE',
            title=f"{'Specialize' if hierarchy['depth'] > 2 else 'Elevate'} {hierarchy['prefix']}* ({hierarchy['count']} items)",
            description=f"Hierarchy: {hierarchy['count']} files share prefix {hierarchy['prefix']} at depth {hierarchy['depth']}",
            action="Create parent interface or specialize implementations",
            source_patterns=[f"hierarchy:{hierarchy['prefix']}"],
            confidence=0.7,
            layer=layers[0] if layers else 'L1',
            semantic_category='ACTION',
            implementation_hint=f"Files: {files_hint[:60]}..." if files_hint else f"Layers: {layer_str}"
        )
        self.ideas.append(idea)
        return idea

    def generate_from_isolation(self, isolated_nodes: List[str]) -> List[DataIdea]:
        """Generate ideas from isolated nodes."""
        ideas = []
        for hex_id in isolated_nodes[:5]:
            idea = DataIdea(
                idea_id=self._generate_id(),
                idea_type='BRIDGE',
                title=f"Connect isolated node {hex_id[:8]}",
                description=f"Node {hex_id} has no connections to ecosystem",
                action="Create relationship edges or remove orphan",
                source_patterns=[f"isolated:{hex_id}"],
                confidence=0.6,
                layer='L3',
                semantic_category='RELATION',
                implementation_hint="Find semantically similar nodes to connect"
            )
            ideas.append(idea)
            self.ideas.append(idea)
        return ideas

    def generate_from_bridge(self, bridges: List[Tuple[str, str]],
                              nodes: Dict = None) -> List[DataIdea]:
        """Generate ideas from bridge edges."""
        ideas = []
        for from_hex, to_hex in bridges[:5]:
            from_cat = 'unknown'
            to_cat = 'unknown'
            if nodes:
                from_node = nodes.get(from_hex)
                to_node = nodes.get(to_hex)
                if from_node:
                    from_cat = from_node.semantic_category
                if to_node:
                    to_cat = to_node.semantic_category

            idea = DataIdea(
                idea_id=self._generate_id(),
                idea_type='BRIDGE',
                title=f"Bridge: {from_cat} -> {to_cat}",
                description=f"Cross-category edge: {from_hex[:8]} ({from_cat}) <-> {to_hex[:8]} ({to_cat})",
                action="Document dependency or add redundant path",
                source_patterns=[f"bridge:{from_hex}:{to_hex}"],
                confidence=0.8,
                layer='L9',
                semantic_category='RELATION',
                implementation_hint=f"Critical link between {from_cat} and {to_cat} domains"
            )
            ideas.append(idea)
            self.ideas.append(idea)
        return ideas


class DataAnalysisShell:
    """
    Data Analysis Shell - Bidirectional Pattern Detection & Idea Generation.

    HEX_ID: 44415441 (DATA)

    Usage:
        analyzer = DataAnalysisShell()
        analyzer.analyze_tokens(tokens)
        patterns = analyzer.get_patterns()
        ideas = analyzer.get_ideas()
    """

    def __init__(self):
        self.hex_id = '44415441'
        self.structure_discovery = DataStructureDiscovery()
        self.ecosystem_analyzer = EcosystemRelationshipAnalyzer()
        self.idea_generator = IdeaGenerator()
        self.patterns: List[DataPattern] = []
        self.tokens_analyzed = 0

    def analyze_tokens(self, tokens: List[Dict]) -> Dict:
        """
        Analyze a list of tokens for patterns and ideas.

        Args:
            tokens: List of token dicts with hex_id, morton, basin, layer, topics

        Returns:
            Analysis summary
        """
        self.tokens_analyzed = len(tokens)

        # Discover structures
        clusters = self.structure_discovery.discover_clusters(tokens)
        hierarchies = self.structure_discovery.discover_hierarchies(tokens)
        sequences = self.structure_discovery.discover_sequences(tokens)

        # Build ecosystem graph
        for token in tokens:
            hex_id = token.get('hex_id', '')
            morton = token.get('morton', 0)
            basin = morton % PRIME_131
            layer = token.get('layer', 'L1')
            self.ecosystem_analyzer.add_node(
                hex_id=hex_id,
                node_type='token',
                basin=basin,
                layer=str(layer),
                metadata=token
            )

        # Build edges from shared properties
        topic_index = defaultdict(list)
        layer_index = defaultdict(list)

        for token in tokens:
            hex_id = token.get('hex_id', '')
            layer = str(token.get('layer', 'L1'))
            topics = token.get('topics', [])

            layer_index[layer].append(hex_id)
            for topic in topics[:3]:
                topic_index[topic].append(hex_id)

        # Create edges
        edges_added = 0
        max_edges = 5000
        for topic, hex_ids in topic_index.items():
            if len(hex_ids) >= 2 and len(hex_ids) <= 20:
                for i, h1 in enumerate(hex_ids[:10]):
                    for h2 in hex_ids[i+1:i+4]:
                        if edges_added < max_edges:
                            self.ecosystem_analyzer.add_edge(h1, h2, f'topic:{topic}')
                            edges_added += 1

        for layer, hex_ids in layer_index.items():
            if len(hex_ids) >= 2:
                sample = hex_ids[:20]
                for i in range(0, len(sample) - 1, 2):
                    if edges_added < max_edges:
                        self.ecosystem_analyzer.add_edge(sample[i], sample[i+1], f'layer:{layer}')
                        edges_added += 1

        # Generate ideas
        ideas = []
        for cluster in clusters:
            ideas.append(self.idea_generator.generate_from_cluster(cluster))
        for hierarchy in hierarchies:
            ideas.append(self.idea_generator.generate_from_hierarchy(hierarchy))

        isolated = self.ecosystem_analyzer.find_isolated_nodes()
        if isolated:
            ideas.extend(self.idea_generator.generate_from_isolation(isolated))

        bridges = self.ecosystem_analyzer.find_bridges()
        if bridges:
            ideas.extend(self.idea_generator.generate_from_bridge(
                bridges, nodes=self.ecosystem_analyzer.nodes
            ))

        return {
            'tokens_analyzed': len(tokens),
            'clusters': len(clusters),
            'hierarchies': len(hierarchies),
            'sequences': len(sequences),
            'ideas_generated': len(ideas),
            'isolated_nodes': len(isolated),
        }

    def get_patterns(self) -> List[Dict]:
        """Get discovered patterns."""
        return self.structure_discovery.discovered_structures

    def get_ideas(self) -> List[DataIdea]:
        """Get generated ideas."""
        return self.idea_generator.ideas

    def get_ecosystem_analysis(self) -> Dict:
        """Get ecosystem connectivity analysis."""
        return self.ecosystem_analyzer.analyze_connectivity()

    def get_status(self, json_output: bool = False) -> Dict:
        """Get shell status."""
        status = {
            'hex_id': self.hex_id,
            'protocol': 'CG:HEX|V1|AGENT:44415441',
            'numpy_available': NUMPY_AVAILABLE,
            'patterns_discovered': len(self.structure_discovery.discovered_structures),
            'ideas_generated': len(self.idea_generator.ideas),
            'ecosystem_nodes': len(self.ecosystem_analyzer.nodes),
            'ecosystem_edges': len(self.ecosystem_analyzer.edges),
            'tokens_analyzed': self.tokens_analyzed,
        }

        if json_output:
            return json.dumps(status, indent=2)
        return status

    def run(self, command: str, *args, as_json: bool = False) -> Any:
        """Run a shell command (CLI interface)."""
        commands = {
            'status': lambda: self.get_status(),
            'patterns': lambda: self.get_patterns(),
            'ideas': lambda: [i.to_dict() if hasattr(i, 'to_dict') else asdict(i) for i in self.get_ideas()],
            'ecosystem': lambda: self.get_ecosystem_analysis(),
        }
        if command in commands:
            result = commands[command]()
            if as_json and isinstance(result, (dict, list)):
                return json.dumps(result, indent=2, default=str)
            return result
        return {'error': f'Unknown command: {command}', 'available': list(commands.keys())}
