"""
Hashing utilities for deduplication
"""

import hashlib
from typing import Set, List

class DeduplicatorHash:
    """Hash-based deduplicator for URLs and content"""
    
    def __init__(self):
        self.seen_hashes: Set[str] = set()
    
    def hash_content(self, content: str) -> str:
        """Generate SHA256 hash of content"""
        return hashlib.sha256(content.encode('utf-8')).hexdigest()
    
    def is_duplicate(self, content: str) -> bool:
        """Check if content is duplicate based on hash"""
        content_hash = self.hash_content(content)
        
        if content_hash in self.seen_hashes:
            return True
        
        self.seen_hashes.add(content_hash)
        return False
    
    def deduplicate_list(self, items: List[str]) -> List[str]:
        """Deduplicate list of strings"""
        unique_items = []
        
        for item in items:
            if not self.is_duplicate(item):
                unique_items.append(item)
        
        return unique_items
    
    def get_stats(self) -> dict:
        """Get deduplication statistics"""
        return {
            "unique_items": len(self.seen_hashes)
        }