"""
ChunkHive Multi-Format File Processor
======================================

Universal file processing engine that handles Python, documentation, configuration,
and text files with semantic understanding and hierarchical structure preservation.

This module provides the main interface for processing individual files or entire
repositories, routing each file type to appropriate specialized parsers and
maintaining consistent metadata across all chunk types.

Author: ChunkHive
License: Apache 2.0
Version: 0.2.0
"""

from pathlib import Path
from typing import List, Dict, Optional, Set
import json
import logging
from dataclasses import dataclass

# Import new schema names (with backward compatibility)
from .schema import (
    SemanticChunk,
    ASTNodeMetadata,
    SourceLocation,
    HierarchyMetadata,
    SemanticChunkType,
    # Backward compatibility
    CodeChunk,
    ChunkAST,
    ChunkSpan,
    ChunkHierarchy,
)
from .hybrid_parser import HierarchicalChunker
from .text_parser import chunk_document as chunk_markdown_file


# Configure logging
logger = logging.getLogger(__name__)


# =============================================================================
# FILE TYPE DEFINITIONS
# =============================================================================

@dataclass
class FileTypeInfo:
    """Information about a specific file type and how to process it."""
    extensions: Set[str]
    language: str
    chunk_type: SemanticChunkType
    description: str


# Supported file types mapping
FILE_TYPE_REGISTRY = {
    'python': FileTypeInfo(
        extensions={'.py', '.pyw'},
        language='python',
        chunk_type='module',
        description='Python source files'
    ),
    'documentation': FileTypeInfo(
        extensions={'.md', '.mdx', '.rst', '.txt'},
        language='markdown',
        chunk_type='documentation',
        description='Documentation and text files'
    ),
    'json': FileTypeInfo(
        extensions={'.json'},
        language='json',
        chunk_type='configuration',
        description='JSON configuration files'
    ),
    'yaml': FileTypeInfo(
        extensions={'.yaml', '.yml'},
        language='yaml',
        chunk_type='configuration',
        description='YAML configuration files'
    ),
    'toml': FileTypeInfo(
        extensions={'.toml'},
        language='toml',
        chunk_type='configuration',
        description='TOML configuration files'
    ),
}


# =============================================================================
# MAIN FILE PROCESSING ENGINE
# =============================================================================

class UniversalFileProcessor:
    """
    Universal file processing engine for semantic code chunking.
    
    This class serves as the main entry point for processing any file type,
    automatically routing to specialized parsers based on file extension and
    content. Maintains consistent metadata and error handling across all file types.
    
    The processor handles:
    - Python source code (via AST + Tree-sitter hybrid parsing)
    - Documentation files (Markdown, RST, plain text)
    - Configuration files (JSON, YAML, TOML)
    - Special files (requirements.txt, Dockerfile, etc.)
    - Unknown/binary files (with graceful fallback)
    
    Attributes:
        use_hierarchical: Whether to use hierarchical AST parsing for Python files
        hierarchical_parser: The hierarchical AST parser instance (if enabled)
        files_processed: Counter of successfully processed files
        files_failed: Counter of failed file processing attempts
        
    Example:
        >>> processor = UniversalFileProcessor()
        >>> chunks = processor.process_file(Path("src/auth.py"))
        >>> print(f"Generated {len(chunks)} chunks")
    """
    
    def __init__(self, use_hierarchical: bool = True):
        """
        Initialize the universal file processor.
        
        Args:
            use_hierarchical: Enable hierarchical AST parsing for Python files.
                             If False, falls back to simple text chunking.
        """
        self.use_hierarchical = use_hierarchical
        self.hierarchical_parser = None
        
        if use_hierarchical:
            try:
                self.hierarchical_parser = HierarchicalChunker()
                logger.info("Hierarchical parser initialized successfully")
            except Exception as e:
                logger.warning(f"Could not initialize hierarchical parser: {e}")
                self.use_hierarchical = False
        
        # Statistics tracking
        self.files_processed = 0
        self.files_failed = 0
        self._supported_extensions = self._build_extension_map()
    
    def _build_extension_map(self) -> Dict[str, str]:
        """Build a mapping of file extensions to file type categories."""
        ext_map = {}
        for category, info in FILE_TYPE_REGISTRY.items():
            for ext in info.extensions:
                ext_map[ext] = category
        return ext_map
    
    def process_file(
        self, 
        file_path: Path, 
        repository_context: Optional[Dict] = None
    ) -> List[SemanticChunk]:
        """
        Process a single file and generate semantic chunks.
        
        This is the main entry point for file processing. It automatically detects
        the file type and routes to the appropriate specialized processor.
        
        Args:
            file_path: Path to the file to process
            repository_context: Optional dictionary containing repository-level
                              metadata (git info, dependencies, etc.)
        
        Returns:
            List of SemanticChunk objects extracted from the file.
            Returns empty list if file cannot be processed.
            
        Example:
            >>> processor = UniversalFileProcessor()
            >>> repo_context = {"repo_url": "https://github.com/user/repo"}
            >>> chunks = processor.process_file(
            ...     Path("src/main.py"),
            ...     repository_context=repo_context
            ... )
        """
        if not file_path.exists():
            logger.error(f"File does not exist: {file_path}")
            self.files_failed += 1
            return []
        
        if not file_path.is_file():
            logger.warning(f"Path is not a file: {file_path}")
            self.files_failed += 1
            return []
        
        try:
            suffix = file_path.suffix.lower()
            file_name = file_path.name.lower()
            
            # Route to appropriate processor based on file type
            chunks = self._route_to_processor(file_path, suffix, file_name, repository_context)
            
            if chunks:
                self.files_processed += 1
                logger.debug(f"Successfully processed {file_path}: {len(chunks)} chunks")
            else:
                logger.debug(f"No chunks generated from {file_path}")
            
            return chunks
            
        except Exception as e:
            logger.error(f"Error processing {file_path}: {e}", exc_info=True)
            self.files_failed += 1
            return []
    
    def _route_to_processor(
        self,
        file_path: Path,
        suffix: str,
        file_name: str,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """Route file to appropriate processor based on type."""
        
        # Python files - use hierarchical AST parser
        if suffix in {'.py', '.pyw'}:
            return self._process_python_file(file_path, repo_context)
        
        # Documentation files
        elif suffix in {'.md', '.mdx', '.rst'}:
            return self._process_documentation_file(file_path, repo_context)
        
        # Configuration files
        elif suffix == '.json':
            return self._process_json_file(file_path, repo_context)
        elif suffix in {'.yaml', '.yml'}:
            return self._process_yaml_file(file_path, repo_context)
        elif suffix == '.toml':
            return self._process_toml_file(file_path, repo_context)
        
        # Special files by name
        elif file_name in {'requirements.txt', 'dockerfile', 'docker-compose.yml'}:
            return self._process_special_file(file_path, repo_context)
        
        # README/LICENSE files
        elif any(keyword in file_name for keyword in ['readme', 'license']):
            return self._process_readme_file(file_path, repo_context)
        
        # Plain text files
        elif suffix in {'.txt', '.cfg', '.ini', '.conf', '.log'}:
            return self._process_text_file(file_path, repo_context)
        
        # Unknown/unsupported files
        else:
            return self._process_unknown_file(file_path, repo_context)
    
    # =========================================================================
    # SPECIALIZED FILE PROCESSORS
    # =========================================================================
    
    def _process_python_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """
        Process Python source files using hierarchical AST parsing.
        
        Uses the HierarchicalChunker to parse Python files with full AST
        analysis, preserving class/function structure and relationships.
        """
        try:
            if self.use_hierarchical and self.hierarchical_parser:
                chunks = self.hierarchical_parser.chunk_file(file_path)
            else:
                logger.debug(f"Falling back to text chunking for {file_path}")
                return self._process_text_file(file_path, repo_context)
            
            # Enrich chunks with repository context
            if repo_context:
                for chunk in chunks:
                    self._add_repository_context(chunk, repo_context)
            
            return chunks
            
        except Exception as e:
            logger.warning(f"Error in hierarchical chunking of {file_path}: {e}")
            return self._process_text_file(file_path, repo_context)
    
    def _process_documentation_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """
        Process documentation files (Markdown, RST).
        
        Uses specialized documentation parser that understands sections,
        code blocks, and documentation structure.
        """
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            # Use documentation-specific parser
            doc_chunks = chunk_markdown_file(
                content,
                source_name=file_path.name,
                source_url=f"file://{file_path.absolute()}"
            )
            
            # Convert to SemanticChunk format
            semantic_chunks = []
            for doc_chunk in doc_chunks:
                chunk = self._convert_doc_chunk_to_semantic(
                    doc_chunk,
                    file_path,
                    repo_context
                )
                semantic_chunks.append(chunk)
            
            return semantic_chunks
            
        except Exception as e:
            logger.error(f"Error processing documentation {file_path}: {e}")
            return self._process_text_file(file_path, repo_context)
    
    def _process_json_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """Process JSON configuration files."""
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            # Parse and pretty-print JSON
            data = json.loads(content)
            pretty_content = json.dumps(data, indent=2, ensure_ascii=False)
            
            chunk = SemanticChunk(
                chunk_id=self._generate_chunk_id("config", file_path, pretty_content),
                file_path=str(file_path),
                language="json",
                chunk_type="configuration",
                code=pretty_content,
                ast=ASTNodeMetadata(
                    symbol_type="configuration",
                    name=file_path.name,
                ),
                span=SourceLocation(
                    start_line=1,
                    end_line=len(pretty_content.split('\n')),
                    char_count=len(pretty_content)
                ),
                metadata={
                    "file_type": "json_configuration",
                    "config_keys": list(data.keys()) if isinstance(data, dict) else [],
                    "is_valid_json": True,
                },
                hierarchy=HierarchyMetadata(
                    is_primary=True,
                    depth=0
                )
            )
            
            if repo_context:
                self._add_repository_context(chunk, repo_context)
            
            return [chunk]
            
        except json.JSONDecodeError as e:
            logger.error(f"Invalid JSON in {file_path}: {e}")
            return self._process_text_file(file_path, repo_context)
        except Exception as e:
            logger.error(f"Error processing JSON {file_path}: {e}")
            return []
    
    def _process_yaml_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """Process YAML configuration files."""
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            chunk = SemanticChunk(
                chunk_id=self._generate_chunk_id("config", file_path, content),
                file_path=str(file_path),
                language="yaml",
                chunk_type="configuration",
                code=content,
                ast=ASTNodeMetadata(
                    symbol_type="configuration",
                    name=file_path.name,
                ),
                span=SourceLocation(
                    start_line=1,
                    end_line=len(content.split('\n')),
                    char_count=len(content)
                ),
                metadata={
                    "file_type": "yaml_configuration",
                },
                hierarchy=HierarchyMetadata(
                    is_primary=True,
                    depth=0
                )
            )
            
            if repo_context:
                self._add_repository_context(chunk, repo_context)
            
            return [chunk]
            
        except Exception as e:
            logger.error(f"Error processing YAML {file_path}: {e}")
            return []
    
    def _process_toml_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """Process TOML configuration files."""
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            chunk = SemanticChunk(
                chunk_id=self._generate_chunk_id("config", file_path, content),
                file_path=str(file_path),
                language="toml",
                chunk_type="configuration",
                code=content,
                ast=ASTNodeMetadata(
                    symbol_type="configuration",
                    name=file_path.name,
                ),
                span=SourceLocation(
                    start_line=1,
                    end_line=len(content.split('\n')),
                    char_count=len(content)
                ),
                metadata={
                    "file_type": "toml_configuration",
                },
                hierarchy=HierarchyMetadata(
                    is_primary=True,
                    depth=0
                )
            )
            
            if repo_context:
                self._add_repository_context(chunk, repo_context)
            
            return [chunk]
            
        except Exception as e:
            logger.error(f"Error processing TOML {file_path}: {e}")
            return []
    
    def _process_special_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """
        Process special files like requirements.txt, Dockerfile, etc.
        
        These files have specific semantics that we want to preserve
        (e.g., dependencies in requirements.txt).
        """
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            file_name = file_path.name.lower()
            
            # Determine file type and language
            if 'requirements' in file_name:
                language = "pip"
                chunk_type = "configuration"
                extra_metadata = {
                    "dependencies": self._extract_python_dependencies(content)
                }
            elif 'docker' in file_name:
                language = "dockerfile"
                chunk_type = "script"
                extra_metadata = {}
            else:
                language = "text"
                chunk_type = "text"
                extra_metadata = {}
            
            chunk = SemanticChunk(
                chunk_id=self._generate_chunk_id("special", file_path, content),
                file_path=str(file_path),
                language=language,
                chunk_type=chunk_type,
                code=content,
                ast=ASTNodeMetadata(
                    symbol_type=chunk_type,
                    name=file_path.name,
                ),
                span=SourceLocation(
                    start_line=1,
                    end_line=len(content.split('\n')),
                    char_count=len(content)
                ),
                metadata={
                    "file_type": file_name,
                    **extra_metadata
                },
                hierarchy=HierarchyMetadata(
                    is_primary=True,
                    depth=0
                )
            )
            
            if repo_context:
                self._add_repository_context(chunk, repo_context)
            
            return [chunk]
            
        except Exception as e:
            logger.error(f"Error processing special file {file_path}: {e}")
            return []
    
    def _process_readme_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """Process README and LICENSE files."""
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            file_name = file_path.name.lower()
            language = "markdown" if file_path.suffix in {'.md', '.mdx'} else "text"
            
            chunk = SemanticChunk(
                chunk_id=self._generate_chunk_id("readme", file_path, content),
                file_path=str(file_path),
                language=language,
                chunk_type="documentation",
                code=content,
                ast=ASTNodeMetadata(
                    symbol_type="documentation",
                    name=file_path.name,
                ),
                span=SourceLocation(
                    start_line=1,
                    end_line=len(content.split('\n')),
                    char_count=len(content)
                ),
                metadata={
                    "file_type": "readme_or_license",
                    "is_readme": "readme" in file_name,
                    "is_license": "license" in file_name,
                },
                hierarchy=HierarchyMetadata(
                    is_primary=True,
                    depth=0
                )
            )
            
            if repo_context:
                self._add_repository_context(chunk, repo_context)
            
            return [chunk]
            
        except Exception as e:
            logger.error(f"Error processing README/LICENSE {file_path}: {e}")
            return []
    
    def _process_text_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict],
        max_lines_per_chunk: int = 200
    ) -> List[SemanticChunk]:
        """
        Process plain text files.
        
        Large text files are split into multiple chunks for better
        embedding and retrieval performance.
        """
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            lines = content.split('\n')
            
            # Small files: single chunk
            if len(lines) <= max_lines_per_chunk:
                chunk = self._create_text_chunk(
                    content, file_path, repo_context, chunk_index=0
                )
                return [chunk]
            
            # Large files: multiple chunks
            chunks = []
            chunk_size = max_lines_per_chunk
            
            for i in range(0, len(lines), chunk_size):
                chunk_lines = lines[i:i + chunk_size]
                chunk_content = '\n'.join(chunk_lines)
                
                chunk = self._create_text_chunk(
                    chunk_content,
                    file_path,
                    repo_context,
                    chunk_index=i // chunk_size,
                    start_line=i + 1,
                    end_line=min(i + chunk_size, len(lines))
                )
                chunks.append(chunk)
            
            return chunks
            
        except Exception as e:
            logger.error(f"Error processing text file {file_path}: {e}")
            return []
    
    def _process_unknown_file(
        self,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> List[SemanticChunk]:
        """
        Handle unknown or unsupported file types.
        
        Attempts to read as text; skips if binary.
        """
        try:
            content = self._read_file_safely(file_path)
            if content is None:
                return []
            
            # Check if content looks like binary data
            if self._is_likely_binary(content):
                logger.debug(f"Skipping binary file: {file_path}")
                return []
            
            # If it's readable text, process as text file
            return self._process_text_file(file_path, repo_context)
            
        except UnicodeDecodeError:
            logger.debug(f"Skipping binary file (decode error): {file_path}")
            return []
        except Exception as e:
            logger.warning(f"Error with unknown file {file_path}: {e}")
            return []
    
    # =========================================================================
    # HELPER METHODS
    # =========================================================================
    
    def _read_file_safely(self, file_path: Path) -> Optional[str]:
        """Safely read file content with proper encoding handling."""
        try:
            return file_path.read_text(encoding='utf-8', errors='ignore')
        except Exception as e:
            logger.error(f"Could not read file {file_path}: {e}")
            return None
    
    def _generate_chunk_id(self, prefix: str, file_path: Path, content: str) -> str:
        """Generate a deterministic chunk ID."""
        content_hash = hash(content) % 10000
        return f"{prefix}_{file_path.stem}_{content_hash:04d}"
    
    def _create_text_chunk(
        self,
        content: str,
        file_path: Path,
        repo_context: Optional[Dict],
        chunk_index: int = 0,
        start_line: int = 1,
        end_line: Optional[int] = None
    ) -> SemanticChunk:
        """Create a semantic chunk for text content."""
        if end_line is None:
            end_line = len(content.split('\n'))
        
        chunk = SemanticChunk(
            chunk_id=f"text_{file_path.stem}_{chunk_index:03d}",
            file_path=str(file_path),
            language="text",
            chunk_type="text",
            code=content,
            ast=ASTNodeMetadata(
                symbol_type="text",
                name=file_path.name,
            ),
            span=SourceLocation(
                start_line=start_line,
                end_line=end_line,
                char_count=len(content)
            ),
            metadata={
                "file_type": "plain_text",
                "chunk_index": chunk_index,
                "total_lines": end_line - start_line + 1,
            },
            hierarchy=HierarchyMetadata(
                is_primary=True,
                depth=0,
                sibling_index=chunk_index
            )
        )
        
        if repo_context:
            self._add_repository_context(chunk, repo_context)
        
        return chunk
    
    def _add_repository_context(
        self,
        chunk: SemanticChunk,
        repo_context: Dict
    ) -> None:
        """Add repository context metadata to a chunk."""
        if "repo_info" not in chunk.metadata:
            chunk.metadata["repo_info"] = {}
        chunk.metadata["repo_info"].update(repo_context)
    
    def _convert_doc_chunk_to_semantic(
        self,
        doc_chunk: Dict,
        file_path: Path,
        repo_context: Optional[Dict]
    ) -> SemanticChunk:
        """Convert documentation chunk to SemanticChunk format."""
        chunk = SemanticChunk(
            chunk_id=doc_chunk.get("chunk_id", self._generate_chunk_id("doc", file_path, doc_chunk["content"])),
            file_path=str(file_path),
            language=doc_chunk.get("language", "markdown"),
            chunk_type="documentation",
            code=doc_chunk["content"],
            ast=ASTNodeMetadata(
                symbol_type="documentation",
                name=file_path.name,
            ),
            span=SourceLocation(
                start_line=doc_chunk.get("metadata", {}).get("line_start", 1),
                end_line=doc_chunk.get("metadata", {}).get("line_end", 1),
            ),
            metadata={
                "doc_chunk_type": doc_chunk.get("chunk_type", "text"),
                **doc_chunk.get("metadata", {})
            },
            hierarchy=HierarchyMetadata(
                is_primary=True,
                depth=0
            )
        )
        
        if repo_context:
            self._add_repository_context(chunk, repo_context)
        
        return chunk
    
    def _extract_python_dependencies(self, requirements_content: str) -> List[str]:
        """
        Extract package names from requirements.txt content.
        
        Parses lines like:
        - package==1.0.0
        - package>=1.0.0
        - package[extra]>=1.0.0
        """
        dependencies = []
        for line in requirements_content.split('\n'):
            line = line.strip()
            
            # Skip empty lines and comments
            if not line or line.startswith('#'):
                continue
            
            # Handle -r includes (requirements includes)
            if line.startswith('-r '):
                continue
            
            # Extract package name (before version specifiers and extras)
            package = line.split('==')[0].split('>=')[0].split('<=')[0].split('[')[0].strip()
            
            if package:
                dependencies.append(package)
        
        return dependencies
    
    def _is_likely_binary(self, content: str, threshold: float = 0.3) -> bool:
        """
        Determine if content appears to be binary data.
        
        Uses ratio of printable to non-printable characters as heuristic.
        """
        if not content:
            return False
        
        # Count printable characters (ASCII 32-126 plus common whitespace)
        printable_count = sum(
            1 for c in content 
            if 32 <= ord(c) <= 126 or c in '\n\r\t'
        )
        
        total_count = len(content)
        if total_count == 0:
            return False
        
        printable_ratio = printable_count / total_count
        return printable_ratio < threshold
    
    def get_statistics(self) -> Dict[str, int]:
        """Get processing statistics."""
        return {
            "files_processed": self.files_processed,
            "files_failed": self.files_failed,
            "total_attempts": self.files_processed + self.files_failed
        }
    
    # =========================================================================
    # BACKWARD COMPATIBILITY METHODS
    # =========================================================================
    
    def chunk_file(
        self,
        file_path: Path,
        repo_metadata: Optional[Dict] = None
    ) -> List[SemanticChunk]:
        """
        DEPRECATED: Use process_file() instead.
        
        This method exists for backward compatibility with existing code.
        It simply calls process_file() with the same arguments.
        
        Args:
            file_path: Path to file to process
            repo_metadata: Optional repository metadata
            
        Returns:
            List of SemanticChunk objects
        """
        return self.process_file(file_path, repository_context=repo_metadata)


# =============================================================================
# BACKWARD COMPATIBILITY ALIAS
# =============================================================================

# Keep old name for backward compatibility
RepoChunker = UniversalFileProcessor


# =============================================================================
# MODULE EXPORTS
# =============================================================================

__all__ = [
    "UniversalFileProcessor",
    "FileTypeInfo",
    "FILE_TYPE_REGISTRY",
    # Backward compatibility
    "RepoChunker",
]



# from pathlib import Path
# from typing import List, Dict, Optional, cast
# import json
# import yaml
# import re
# from .hybrid_parser import HierarchicalChunker
# from .schema import CodeChunk, ChunkAST, ChunkSpan, ChunkHierarchy, ChunkType, ASTSymbolType
# from .text_parser import chunk_document as chunk_markdown_file


# class RepoChunker:
#     """
#     Repository chunker that handles ALL file types with proper structure
#     """
    
#     def __init__(self, use_hierarchical: bool = True):
#         if use_hierarchical:
#             self.hierarchical_chunker = HierarchicalChunker()
#         self.use_hierarchical = use_hierarchical
    
#     def chunk_file(self, file_path: Path, repo_metadata: Optional[Dict] = None) -> List[CodeChunk]:
#         """
#         Chunk ANY file type with repository context
        
#         Args:
#             file_path: Path to the file
#             repo_metadata: Optional dict with repo metadata
#         """
#         suffix = file_path.suffix.lower()
        
#         # Python files - use your advanced hierarchical chunker
#         if suffix == '.py':
#             return self._chunk_python_file(file_path, repo_metadata)
        
#         # Markdown/RST documentation
#         elif suffix in ['.md', '.mdx', '.rst']:
#             return self._chunk_markdown_file_wrapper(file_path, repo_metadata)
        
#         # JSON config files
#         elif suffix == '.json':
#             return self._chunk_json_file(file_path, repo_metadata)
        
#         # YAML/TOML config files
#         elif suffix in ['.yaml', '.yml', '.toml']:
#             return self._chunk_config_file(file_path, repo_metadata)
        
#         # Requirements/Docker files
#         elif file_path.name.lower() in ['requirements.txt', 'dockerfile', 'docker-compose.yml']:
#             return self._chunk_special_file(file_path, repo_metadata)
        
#         # Text files
#         elif suffix in ['.txt', '.cfg', '.ini', '.conf']:
#             return self._chunk_text_file(file_path, repo_metadata)
        
#         # README/LICENSE files
#         elif file_path.name.lower() in ['readme', 'readme.md', 'license', 'license.txt', 'license.md']:
#             return self._chunk_readme_file(file_path, repo_metadata)
        
#         # All other files
#         else:
#             return self._chunk_other_file(file_path, repo_metadata)
    
#     def _chunk_python_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Use our hierarchical chunker for Python files"""
#         try:
#             if self.use_hierarchical:
#                 chunks = self.hierarchical_chunker.chunk_file(file_path)
#             else:
#                 # Fallback to hybrid chunking
#                 # Fallback to basic text chunking instead of hybrid
#                 return self._chunk_text_file(file_path, repo_metadata)
            
#             # Add repository metadata
#             if repo_metadata:
#                 for chunk in chunks:
#                     if "repo_info" not in chunk.metadata:
#                         chunk.metadata["repo_info"] = {}
#                     chunk.metadata["repo_info"].update(repo_metadata)
            
#             return chunks
            
#         except Exception as e:
#             print(f"⚠️ Error chunking Python file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_markdown_file_wrapper(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Chunk markdown files using our doc_chunker"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
            
#             # Use your existing doc_chunker
#             doc_chunks = chunk_markdown_file(
#                 content, 
#                 source_name=file_path.name,
#                 source_url=f"file://{file_path}"
#             )
            
#             # Convert to CodeChunk schema
#             code_chunks = []
#             for doc_chunk in doc_chunks:
#                 code_chunk = CodeChunk(
#                     chunk_id=doc_chunk["chunk_id"],
#                     file_path=str(file_path),
#                     language=doc_chunk.get("language", "markdown"),
#                     chunk_type="documentation",  # ✅ Fixed type
#                     code=doc_chunk["content"],
#                     ast=ChunkAST(
#                         symbol_type="documentation",  # ✅ Fixed type
#                         name=file_path.name,
#                         parent=None,
#                         docstring=None
#                     ),
#                     span=ChunkSpan(
#                         start_line=doc_chunk.get("metadata", {}).get("line_start", 1),
#                         end_line=doc_chunk.get("metadata", {}).get("line_end", 1)
#                     ),
#                     metadata={
#                         "doc_chunk_type": doc_chunk.get("chunk_type", "text"),
#                         "repo_info": repo_metadata or {},
#                         **doc_chunk.get("metadata", {})
#                     },
#                     hierarchy=ChunkHierarchy(
#                         is_primary=True,
#                         is_extracted=False,
#                         depth=0
#                     )
#                 )
#                 code_chunks.append(code_chunk)
            
#             return code_chunks
            
#         except Exception as e:
#             print(f"⚠️ Error chunking markdown file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_json_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Chunk JSON config files"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
#             data = json.loads(content)
            
#             pretty_content = json.dumps(data, indent=2)
            
#             chunk = CodeChunk(
#                 chunk_id=f"config_{file_path.stem}_{hash(pretty_content) % 10000:04d}",
#                 file_path=str(file_path),
#                 language="json",
#                 chunk_type="configuration",  # ✅ Fixed type
#                 code=pretty_content,
#                 ast=ChunkAST(
#                     symbol_type="configuration",  # ✅ Fixed type
#                     name=file_path.name,
#                     parent=None,
#                     docstring=None
#                 ),
#                 span=ChunkSpan(
#                     start_line=1,
#                     end_line=len(pretty_content.split('\n'))
#                 ),
#                 metadata={
#                     "file_type": "json_config",
#                     "config_keys": list(data.keys()) if isinstance(data, dict) else [],
#                     "repo_info": repo_metadata or {}
#                 },
#                 hierarchy=ChunkHierarchy(
#                     is_primary=True,
#                     is_extracted=False,
#                     depth=0
#                 )
#             )
            
#             return [chunk]
            
#         except Exception as e:
#             print(f"⚠️ Error chunking JSON file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_config_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Chunk YAML/TOML config files"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
#             suffix = file_path.suffix.lower()
            
#             language = "yaml" if suffix in ['.yaml', '.yml'] else "toml"
            
#             chunk = CodeChunk(
#                 chunk_id=f"config_{file_path.stem}_{hash(content) % 10000:04d}",
#                 file_path=str(file_path),
#                 language=language,
#                 chunk_type="configuration",  # ✅ Fixed type
#                 code=content,
#                 ast=ChunkAST(
#                     symbol_type="configuration",  # ✅ Fixed type
#                     name=file_path.name,
#                     parent=None,
#                     docstring=None
#                 ),
#                 span=ChunkSpan(
#                     start_line=1,
#                     end_line=len(content.split('\n'))
#                 ),
#                 metadata={
#                     "file_type": f"{language}_config",
#                     "repo_info": repo_metadata or {}
#                 },
#                 hierarchy=ChunkHierarchy(
#                     is_primary=True,
#                     is_extracted=False,
#                     depth=0
#                 )
#             )
            
#             return [chunk]
            
#         except Exception as e:
#             print(f"⚠️ Error chunking config file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_special_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Chunk special files (requirements.txt, Dockerfile, etc.)"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
#             file_name = file_path.name.lower()
            
#             if 'requirements' in file_name:
#                 language = "requirements"
#                 chunk_type = "configuration"
#             elif 'docker' in file_name:
#                 language = "dockerfile"
#                 chunk_type = "script"
#             else:
#                 language = "text"
#                 chunk_type = "text"
            
#             chunk = CodeChunk(
#                 chunk_id=f"special_{file_path.stem}_{hash(content) % 10000:04d}",
#                 file_path=str(file_path),
#                 language=language,
#                 chunk_type=chunk_type,
#                 code=content,
#                 ast=ChunkAST(
#                     symbol_type=chunk_type,
#                     name=file_path.name,
#                     parent=None,
#                     docstring=None
#                 ),
#                 span=ChunkSpan(
#                     start_line=1,
#                     end_line=len(content.split('\n'))
#                 ),
#                 metadata={
#                     "file_type": file_name,
#                     "repo_info": repo_metadata or {},
#                     "dependencies": self._extract_dependencies(content) if "requirements" in file_name else []
#                 },
#                 hierarchy=ChunkHierarchy(
#                     is_primary=True,
#                     is_extracted=False,
#                     depth=0
#                 )
#             )
            
#             return [chunk]
            
#         except Exception as e:
#             print(f"⚠️ Error chunking special file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_text_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Chunk plain text files"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
            
#             # Create a single chunk for small files, multiple for large ones
#             if len(content.split('\n')) <= 200:
#                 chunks = [self._create_text_chunk(content, file_path, repo_metadata)]
#             else:
#                 # Split large text files into reasonable chunks
#                 chunks = []
#                 lines = content.split('\n')
#                 chunk_size = 100
                
#                 for i in range(0, len(lines), chunk_size):
#                     chunk_lines = lines[i:i + chunk_size]
#                     chunk_content = '\n'.join(chunk_lines)
                    
#                     chunk = self._create_text_chunk(
#                         chunk_content, 
#                         file_path, 
#                         repo_metadata,
#                         chunk_index=i // chunk_size
#                     )
#                     chunks.append(chunk)
            
#             return chunks
            
#         except Exception as e:
#             print(f"⚠️ Error reading text file {file_path}: {e}")
#             return []
    
#     def _chunk_readme_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Special handling for README/LICENSE files"""
#         try:
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
            
#             chunk = CodeChunk(
#                 chunk_id=f"readme_{file_path.stem}_{hash(content) % 10000:04d}",
#                 file_path=str(file_path),
#                 language="markdown" if file_path.suffix in ['.md', '.mdx'] else "text",
#                 chunk_type="documentation",  # ✅ Fixed type
#                 code=content,
#                 ast=ChunkAST(
#                     symbol_type="documentation",  # ✅ Fixed type
#                     name=file_path.name,
#                     parent=None,
#                     docstring=None
#                 ),
#                 span=ChunkSpan(
#                     start_line=1,
#                     end_line=len(content.split('\n'))
#                 ),
#                 metadata={
#                     "file_type": "readme_license",
#                     "is_readme": "readme" in file_path.name.lower(),
#                     "is_license": "license" in file_path.name.lower(),
#                     "repo_info": repo_metadata or {}
#                 },
#                 hierarchy=ChunkHierarchy(
#                     is_primary=True,
#                     is_extracted=False,
#                     depth=0
#                 )
#             )
            
#             return [chunk]
            
#         except Exception as e:
#             print(f"⚠️ Error chunking README file {file_path}: {e}")
#             return self._chunk_text_file(file_path, repo_metadata)
    
#     def _chunk_other_file(self, file_path: Path, repo_metadata: Optional[Dict]) -> List[CodeChunk]:
#         """Fallback for unknown file types (binary or unsupported)"""
#         try:
#             # Try to read as text first
#             content = file_path.read_text(encoding='utf-8', errors='ignore')
            
#             # If it looks like binary (mostly non-printable characters)
#             if self._looks_like_binary(content):
#                 print(f"⚠️ Skipping binary file: {file_path}")
#                 return []
            
#             # If readable text, treat as text file
#             return self._chunk_text_file(file_path, repo_metadata)
            
#         except UnicodeDecodeError:
#             print(f"⚠️ Skipping binary file: {file_path}")
#             return []
#         except Exception as e:
#             print(f"⚠️ Error with file {file_path}: {e}")
#             return []
    
#     def _create_text_chunk(self, content: str, file_path: Path, 
#                           repo_metadata: Optional[Dict], chunk_index: int = 0) -> CodeChunk:
#         """Helper to create a text chunk"""
#         lines = content.split('\n')
        
#         return CodeChunk(
#             chunk_id=f"text_{file_path.stem}_{chunk_index:03d}",
#             file_path=str(file_path),
#             language="text",
#             chunk_type="text",  # ✅ Fixed type
#             code=content,
#             ast=ChunkAST(
#                 symbol_type="text",  # ✅ Fixed type
#                 name=file_path.name,
#                 parent=None,
#                 docstring=None
#             ),
#             span=ChunkSpan(
#                 start_line=1,
#                 end_line=len(lines)
#             ),
#             metadata={
#                 "file_type": "text",
#                 "chunk_index": chunk_index,
#                 "total_lines": len(lines),
#                 "repo_info": repo_metadata or {}
#             },
#             hierarchy=ChunkHierarchy(
#                 is_primary=True,
#                 is_extracted=False,
#                 depth=0
#             )
#         )
    
#     def _extract_dependencies(self, requirements_content: str) -> List[str]:
#         """Extract package names from requirements.txt"""
#         dependencies = []
#         for line in requirements_content.split('\n'):
#             line = line.strip()
#             if line and not line.startswith('#'):
#                 # Extract package name (before version specifiers)
#                 package = line.split('==')[0].split('>=')[0].split('<=')[0].strip()
#                 if package:
#                     dependencies.append(package)
#         return dependencies
    
#     def _looks_like_binary(self, content: str, threshold: float = 0.3) -> bool:
#         """Check if content looks like binary data"""
#         if not content:
#             return False
        
#         # Count printable vs non-printable characters
#         printable = sum(1 for c in content if 32 <= ord(c) <= 126 or c in '\n\r\t')
#         total = len(content)
        
#         if total == 0:
#             return False
        
#         ratio = printable / total
#         return ratio < threshold

