# tests/test_exotic_data.py
"""
Comprehensive test suite for exotic data handling in the fuzzy matching engine.
Tests various scripts, encodings, and edge cases to ensure robustness.
"""
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
SRC = ROOT / "src"
if str(SRC) not in sys.path:
    sys.path.insert(0, str(SRC))

import pytest
import pandas as pd
import logging

from fmatch.engine import (
    ExoticDataDetector,
    determine_optimal_blocking_strategy,
    detect_optimal_variants,
    build_key,
    BlockingMode,
    smart_fallback,
    ENGINE_CONFIG,
    configure_engine,
)
from fmatch.core.engine import _quick_algo_fitness

log = logging.getLogger(__name__)

class ExoticDataGenerator:
    """Generates test datasets with various exotic characteristics."""
    
    @staticmethod
    def create_thai_business_data() -> pd.DataFrame:
        """Thai business registry with mixed Thai/English."""
        return pd.DataFrame({
            'company_id': ['TH001', 'TH002', 'TH003', 'TH004', 'TH005'],
            'company_name_th': [
                'บริษัท สยามเทค จำกัด',
                'ห้างหุ้นส่วน กรุงเทพการค้า',
                'บริษัท ไทยโกลบอล อินดัสทรี',
                'สำนักงาน นวัตกรรมดิจิทัล',
                'บริษัท อาหารไทย เอ็กซ์พอร์ต'
            ],
            'company_name_en': [
                'Siam Tech Co., Ltd.',
                'Bangkok Trading Partnership',
                'Thai Global Industry Co., Ltd.',
                'Digital Innovation Office',
                'Thai Food Export Co., Ltd.'
            ],
            'address': [
                '123 ถ.สุขุมวิท กรุงเทพฯ 10110',
                '456 ถ.เพชรบุรี กรุงเทพฯ 10400',
                '789 ถ.พระราม 4 กรุงเทพฯ 10500',
                '321 ถ.สีลม กรุงเทพฯ 10500',
                '654 ถ.รัชดาภิเษก กรุงเทพฯ 10900'
            ],
            'phone': [
                '02-123-4567',
                '02-234-5678',
                '02-345-6789',
                '02-456-7890',
                '02-567-8901'
            ]
        })
    
    @staticmethod
    def create_arabic_mixed_products() -> pd.DataFrame:
        """Product catalog with Arabic/English mixed content."""
        return pd.DataFrame({
            'product_id': ['AR001', 'AR002', 'AR003', 'AR004', 'AR005'],
            'product_name_ar': [
                'هاتف ذكي سامسونج',
                'حاسوب محمول ديل',
                'تلفزيون إل جي',
                'ثلاجة هيتاشي',
                'غسالة بوش'
            ],
            'product_name_en': [
                'Samsung Smart Phone',
                'Dell Laptop',
                'LG Television',
                'Hitachi Refrigerator',
                'Bosch Washing Machine'
            ],
            'category': [
                'إلكترونيات/Electronics',
                'كمبيوتر/Computers',
                'أجهزة منزلية/Home Appliances',
                'أجهزة منزلية/Home Appliances',
                'أجهزة منزلية/Home Appliances'
            ],
            'price': [999.99, 1499.99, 799.99, 1299.99, 899.99]
        })
    
    @staticmethod
    def create_emoji_usernames() -> pd.DataFrame:
        """Social media handles with emoji."""
        return pd.DataFrame({
            'user_id': ['EM001', 'EM002', 'EM003', 'EM004', 'EM005'],
            'username': [
                'CoolDude😎',
                '❤️LoveLife❤️',
                'TechGuru💻🚀',
                '🌟SuperStar🌟',
                'FoodLover🍕🍔'
            ],
            'display_name': [
                'John Smith 😎',
                'Sarah ❤️',
                'Mike Chen 💻',
                'Star Jones ⭐',
                'Bob Wilson 🍕'
            ],
            'bio': [
                'Living my best life 😎✨',
                'Spreading love everywhere ❤️💕',
                'Tech enthusiast 💻 Space lover 🚀',
                'Shining bright like a star ⭐✨',
                'Food is life 🍕🍔🌮'
            ]
        })
    
    @staticmethod
    def create_cyrillic_addresses() -> pd.DataFrame:
        """Russian address data."""
        return pd.DataFrame({
            'address_id': ['RU001', 'RU002', 'RU003', 'RU004', 'RU005'],
            'street': [
                'ул. Тверская',
                'пр. Невский',
                'ул. Арбат',
                'пр. Ленина',
                'ул. Красная'
            ],
            'city': [
                'Москва',
                'Санкт-Петербург',
                'Москва',
                'Новосибирск',
                'Краснодар'
            ],
            'postal_code': ['101000', '191186', '119019', '630099', '350000'],
            'full_address': [
                'ул. Тверская, д. 1, Москва, 101000',
                'пр. Невский, д. 28, Санкт-Петербург, 191186',
                'ул. Арбат, д. 10, Москва, 119019',
                'пр. Ленина, д. 5, Новосибирск, 630099',
                'ул. Красная, д. 135, Краснодар, 350000'
            ]
        })
    
    @staticmethod
    def create_cjk_companies() -> pd.DataFrame:
        """Japanese/Chinese/Korean company data."""
        return pd.DataFrame({
            'company_id': ['JP001', 'CN001', 'KR001', 'JP002', 'CN002'],
            'company_name': [
                '株式会社トヨタ自動車',
                '阿里巴巴集团',
                '삼성전자',
                '任天堂株式会社',
                '腾讯控股有限公司'
            ],
            'company_name_romanized': [
                'Toyota Motor Corporation',
                'Alibaba Group',
                'Samsung Electronics',
                'Nintendo Co., Ltd.',
                'Tencent Holdings Limited'
            ],
            'country': ['Japan', 'China', 'Korea', 'Japan', 'China'],
            'industry': ['Automotive', 'E-commerce', 'Electronics', 'Gaming', 'Technology']
        })
    
    @staticmethod
    def create_mojibake_chaos() -> pd.DataFrame:
        """Data with encoding issues."""
        return pd.DataFrame({
            'id': ['MB001', 'MB002', 'MB003', 'MB004', 'MB005'],
            'name': [
                'JosÃ© GarcÃ­a',  # José García
                'FranÃ§ois CafÃ©',  # François Café
                'MÃ¼ller StrauÃŸ',  # Müller Strauß
                'SeÃ±or NiÃ±o',  # Señor Niño
                'CafÃ© â€™s Menu'  # Café's Menu
            ],
            'email': [
                'jose@garcÃ­a.com',
                'francois@cafÃ©.fr',
                'mueller@strauÃŸ.de',
                'senior@niÃ±o.es',
                'info@cafÃ©menu.com'
            ],
            'notes': [
                'Meeting at cafÃ© â€œLe Petitâ€',
                'Invoice â„–12345',
                'Temperature: 25Â°C',
                'Price: â‚¬100',
                'Itâ€™s working!'
            ]
        })
    
    @staticmethod
    def create_extreme_skew() -> pd.DataFrame:
        """99% nulls with some valid data."""
        size = 1000
        data = {
            'id': [f'ID{i:04d}' for i in range(size)],
            'mostly_null': [None] * (size - 10) + ['Valid'] * 10,
            'sparse_names': [None] * (size - 20) + [f'Name_{i}' for i in range(20)],
            'rare_email': [None] * (size - 5) + [
                'test1@example.com',
                'test2@example.com',
                'test3@example.com',
                'test4@example.com',
                'test5@example.com'
            ]
        }
        return pd.DataFrame(data)
    
    @staticmethod
    def create_mixed_quality_data() -> pd.DataFrame:
        """Real-world messy data with various issues."""
        return pd.DataFrame({
            'record_id': ['MQ001', 'MQ002', 'MQ003', 'MQ004', 'MQ005', 'MQ006'],
            'name': [
                'John Smith',
                'JOHN SMITH',
                'Smith, John',
                'J. Smith',
                'john.smith@email.com',
                None
            ],
            'company': [
                'ABC Corp.',
                'ABC Corporation',
                'A.B.C. Corp',
                'ABC CORP',
                'abc corp.',
                'ABC'
            ],
            'phone': [
                '555-123-4567',
                '(555) 123-4567',
                '5551234567',
                '555.123.4567',
                '+1-555-123-4567',
                '555 123 4567'
            ],
            'address': [
                '123 Main St.',
                '123 Main Street',
                '123 Main St',
                '123 MAIN ST.',
                '123 Main Street, Suite 100',
                '123 Main'
            ]
        })


class TestExoticDataDetection:
    """Tests for the ExoticDataDetector."""
    
    def test_script_detection(self):
        """Test detection of various scripts."""
        test_cases = [
            ('Hello World', 'latin', 1.0),
            ('Привет мир', 'cyrillic', 1.0),
            ('مرحبا بالعالم', 'arabic', 1.0),
            ('שלום עולם', 'hebrew', 1.0),
            ('你好世界', 'cjk', 1.0),
            ('สวัสดีโลก', 'thai', 1.0),
            ('Hello 世界', 'mixed', None),  # Mixed scripts
            ('12345', 'unknown', 0.0),  # Numbers only
        ]
        
        for text, expected_script, min_confidence in test_cases:
            result = ExoticDataDetector._detect_script(text)
            assert result[0] == expected_script, f"Failed for '{text}': got {result[0]}, expected {expected_script}"
            if min_confidence is not None:
                assert result[1] >= min_confidence, f"Low confidence for '{text}': {result[1]}"
    
    def test_semantic_type_detection(self):
        """Test semantic type detection."""
        test_data = {
            'email': pd.Series(['test@example.com', 'user@domain.org', 'admin@company.net']),
            'phone': pd.Series(['555-123-4567', '(555) 987-6543', '555.111.2222']),
            'url': pd.Series(['https://example.com', 'www.test.org', 'http://site.net']),
            'postal_code': pd.Series(['12345', '98765-4321', '10001']),
            'product_code': pd.Series(['ABC123', 'XYZ-789', 'PROD456789']),
        }
        
        for expected_type, series in test_data.items():
            result = ExoticDataDetector.detect_characteristics(series)
            assert result['semantic_type'] == expected_type, f"Failed to detect {expected_type}"
    
    def test_encoding_issues_detection(self):
        """Test detection of encoding problems."""
        # Create series with mojibake
        mojibake_series = pd.Series([
            'Normal text',
            'Ã©tÃ©',  # été
            'cafÃ©',  # café
            'â€œquotesâ€',  # "quotes"
            'More normal text'
        ])
        
        result = ExoticDataDetector.detect_characteristics(mojibake_series)
        assert result['encoding_issues'] == True, "Failed to detect encoding issues"
        
        # Clean series should not have issues
        clean_series = pd.Series(['Hello', 'World', 'Test', 'Data'])
        result = ExoticDataDetector.detect_characteristics(clean_series)
        assert result['encoding_issues'] == False, "False positive for encoding issues"
    
    def test_emoji_detection(self):
        """Test emoji detection."""
        emoji_series = pd.Series(['Hello 😊', 'World 🌍', 'Test 🧪', 'Normal', 'Text'])
        result = ExoticDataDetector.detect_characteristics(emoji_series)
        assert result['has_emoji'] == True, "Failed to detect emoji"
        
        no_emoji_series = pd.Series(['Hello', 'World', 'Test', 'Normal', 'Text'])
        result = ExoticDataDetector.detect_characteristics(no_emoji_series)
        assert result['has_emoji'] == False, "False positive for emoji"


class TestBlockingStrategiesWithExoticData:
    """Tests for blocking strategy selection with exotic data."""
    
    def test_thai_blocking_strategy(self):
        """Test blocking on Thai business data."""
        df = ExoticDataGenerator.create_thai_business_data()
        
        # Create simple mappings
        mappings = [
            {'source': 'company_name_th', 'weight': 1.0},
            {'source': 'company_name_en', 'weight': 1.0},
            {'source': 'address', 'weight': 0.8}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        # Should not use phonetic variants for Thai
        assert result['preproc'] not in ['soundex', 'metaphone']
        log.info(f"Thai blocking strategy: {result}")
    
    def test_arabic_blocking_strategy(self):
        """Test blocking on Arabic/English mixed data."""
        df = ExoticDataGenerator.create_arabic_mixed_products()
        
        mappings = [
            {'source': 'product_name_ar', 'weight': 1.0},
            {'source': 'product_name_en', 'weight': 1.0},
            {'source': 'category', 'weight': 0.7}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        # Should handle RTL scripts appropriately
        assert result['preproc'] != 'soundex'  # Soundex doesn't work for Arabic
        log.info(f"Arabic blocking strategy: {result}")
    
    def test_emoji_blocking_strategy(self):
        """Test blocking on data with emoji."""
        df = ExoticDataGenerator.create_emoji_usernames()
        
        mappings = [
            {'source': 'username', 'weight': 1.0},
            {'source': 'display_name', 'weight': 0.9}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        # Should handle or strip emoji appropriately
        log.info(f"Emoji blocking strategy: {result}")
    
    def test_cyrillic_blocking_strategy(self):
        """Test blocking on Cyrillic data."""
        df = ExoticDataGenerator.create_cyrillic_addresses()
        
        mappings = [
            {'source': 'street', 'weight': 1.0},
            {'source': 'city', 'weight': 1.0},
            {'source': 'full_address', 'weight': 0.8}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        log.info(f"Cyrillic blocking strategy: {result}")
    
    def test_cjk_blocking_strategy(self):
        """Test blocking on CJK (Chinese/Japanese/Korean) data."""
        df = ExoticDataGenerator.create_cjk_companies()
        
        mappings = [
            {'source': 'company_name', 'weight': 1.0},
            {'source': 'company_name_romanized', 'weight': 0.9}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        # Should use character-based approaches for CJK
        assert result['preproc'] in ['exact_substring', 'first_n_chars']
        log.info(f"CJK blocking strategy: {result}")
    
    def test_mojibake_fallback(self):
        """Test fallback strategies for data with encoding issues."""
        df = ExoticDataGenerator.create_mojibake_chaos()
        
        mappings = [
            {'source': 'name', 'weight': 1.0},
            {'source': 'email', 'weight': 0.8}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        # Should either apply blocking with unicode normalization or use fallback
        if result['apply_blocking']:
            assert result.get('is_fallback') or result['preproc'] == 'unicode_normalize'
        log.info(f"Mojibake strategy: {result}")
    
    def test_extreme_skew_fallback(self):
        """Test handling of extremely sparse data."""
        df = ExoticDataGenerator.create_extreme_skew()
        
        mappings = [
            {'source': 'mostly_null', 'weight': 1.0},
            {'source': 'sparse_names', 'weight': 0.9},
            {'source': 'rare_email', 'weight': 0.8}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        # Should use smart fallback due to extreme nulls
        assert result.get('is_fallback', False) == True
        if result['apply_blocking']:
            assert 'null' in result.get('fallback_reason', '').lower()
        log.info(f"Extreme skew strategy: {result}")


class TestPreprocessingVariants:
    """Tests for new preprocessing variants."""
    
    def test_unicode_normalize(self):
        """Test Unicode normalization preprocessing."""
        series = pd.Series(['café', 'naïve', 'résumé', 'Zürich'])
        keys, eff_k = build_key(series, 'unicode_normalize', 6, 'default')
        
        # Should normalize accented characters
        assert all(key.isascii() for key in keys if key), "Unicode normalization failed"
    
    def test_exact_substring(self):
        """Test exact substring preprocessing."""
        series = pd.Series(['ABC123', 'DEF456', 'GHI789'])
        keys, eff_k = build_key(series, 'exact_substring', 3, 'default')
        
        assert keys.tolist() == ['ABC', 'DEF', 'GHI']
    
    def test_strip_emoji(self):
        """Test emoji stripping preprocessing."""
        series = pd.Series(['Hello😊', 'World🌍', 'Test🧪'])
        keys, eff_k = build_key(series, 'strip_emoji', 5, 'default')
        
        # Should remove emoji
        assert all('😊' not in key and '🌍' not in key and '🧪' not in key for key in keys)
    
    def test_first_n_chars(self):
        """Test first N characters preprocessing."""
        series = pd.Series(['Hello World', 'Testing 123', 'Sample Text'])
        keys, eff_k = build_key(series, 'first_n_chars', 4, 'default')
        
        assert keys.tolist() == ['Hell', 'Test', 'Samp']


class TestOptimalVariantDetection:
    """Tests for detect_optimal_variants function."""
    
    def test_email_variant_detection(self):
        """Test that email columns get email-specific variants."""
        series = pd.Series(['test@example.com', 'user@domain.org', 'admin@company.net'])
        series.name = 'email_address'
        
        variants = detect_optimal_variants(series, series.name)
        assert 'email_domain' in variants
    
    def test_phone_variant_detection(self):
        """Test that phone columns get phone-specific variants."""
        series = pd.Series(['555-123-4567', '(555) 987-6543', '555.111.2222'])
        series.name = 'phone_number'
        
        variants = detect_optimal_variants(series, series.name)
        assert any(v in variants for v in ['phone_digits', 'phone_area'])
    
    def test_company_variant_detection(self):
        """Test that company columns get appropriate variants."""
        series = pd.Series(['ABC Corporation', 'XYZ Inc.', 'Tech Company LLC'])
        series.name = 'company_name'
        
        variants = detect_optimal_variants(series, series.name)
        assert 'company_stem' in variants
    
    def test_exotic_script_variant_detection(self):
        """Test variant detection for non-Latin scripts."""
        # Arabic
        arabic_series = pd.Series(['مرحبا', 'عالم', 'اختبار'])
        arabic_series.name = 'arabic_text'
        variants = detect_optimal_variants(arabic_series, arabic_series.name)
        assert 'soundex' not in variants  # Soundex doesn't work for Arabic
        assert 'unicode_normalize' in variants
        
        # CJK
        cjk_series = pd.Series(['你好', '世界', '测试'])
        cjk_series.name = 'chinese_text'
        variants = detect_optimal_variants(cjk_series, cjk_series.name)
        assert any(v in variants for v in ['exact_substring', 'first_n_chars'])
        assert 'soundex' not in variants


class TestAlgorithmFitness:
    """Tests for algorithm fitness evaluation."""
    
    @pytest.fixture
    def setup_fitness_testing(self):
        """Enable algorithm fitness for tests."""
        original_config = ENGINE_CONFIG.enable_algorithm_fitness
        configure_engine(enable_algorithm_fitness=True)
        yield
        configure_engine(enable_algorithm_fitness=original_config)
    
    def test_soundex_fitness_for_nonlatin(self, setup_fitness_testing):
        """Test that Soundex gets low fitness for non-Latin text."""
        # Arabic names
        sample_pairs = [
            ('محمد', 'محمد'),
            ('أحمد', 'احمد'),
            ('عبدالله', 'عبد الله')
        ]
        
        column_profile = {
            'script': 'arabic',
            'semantic_type': 'name',
            'avg_length': 6
        }
        
        fitness = _quick_algo_fitness(sample_pairs, 'Soundex', column_profile)
        assert fitness < 0.3, f"Soundex fitness too high for Arabic: {fitness}"
    
    def test_exact_fitness_for_codes(self, setup_fitness_testing):
        """Test that Exact matching gets high fitness for product codes."""
        sample_pairs = [
            ('ABC123', 'ABC123'),
            ('XYZ789', 'XYZ789'),
            ('DEF456', 'DEF456')
        ]
        
        column_profile = {
            'script': 'latin',
            'semantic_type': 'product_code',
            'avg_length': 6
        }
        
        fitness = _quick_algo_fitness(sample_pairs, 'Exact', column_profile)
        assert fitness > 0.8, f"Exact fitness too low for product codes: {fitness}"
    
    def test_wratio_fitness_for_short_strings(self, setup_fitness_testing):
        """Test that WRatio gets low fitness for short strings."""
        sample_pairs = [
            ('AB', 'BA'),
            ('CD', 'DC'),
            ('EF', 'FE')
        ]
        
        column_profile = {
            'script': 'latin',
            'semantic_type': None,
            'avg_length': 2
        }
        
        fitness = _quick_algo_fitness(sample_pairs, 'WRatio', column_profile)
        assert fitness < 0.5, f"WRatio fitness too high for short strings: {fitness}"


class TestSmartFallbacks:
    """Tests for smart fallback strategies."""
    
    def test_truncation_fallback_for_high_uniqueness(self):
        """Test truncation fallback when all columns are too unique."""
        # Create data where every value is unique
        df = pd.DataFrame({
            'uuid': [f'550e8400-e29b-41d4-a716-{i:012d}' for i in range(100)],
            'timestamp': [f'2024-01-01T00:00:{i:02d}' for i in range(100)],
            'random_id': [f'RND{i:010d}' for i in range(100)]
        })
        
        result = smart_fallback(df, BlockingMode.DEDUPE, 'default')
        
        assert result['apply_blocking'] == True
        assert result['is_fallback'] == True
        assert 'truncation' in result['fallback_reason'].lower()
        assert result['preproc'] == 'exact_substring'
    
    def test_composite_fallback_for_low_uniqueness(self):
        """Test composite key fallback when all columns have low uniqueness."""
        # Create data with very few unique values
        df = pd.DataFrame({
            'status': ['Active'] * 50 + ['Inactive'] * 50,
            'type': ['A'] * 30 + ['B'] * 30 + ['C'] * 40,
            'category': ['X'] * 60 + ['Y'] * 40
        })
        
        result = smart_fallback(df, BlockingMode.DEDUPE, 'default')
        
        assert result['is_fallback'] == True
        # Should suggest composite key due to low uniqueness
        assert 'composite' in result['fallback_reason'].lower() or 'truncation' in result['fallback_reason'].lower()
    
    def test_unicode_fallback_for_encoding_chaos(self):
        """Test Unicode normalization fallback for encoding issues."""
        df = ExoticDataGenerator.create_mojibake_chaos()
        
        # Force the fallback by using only the problematic columns
        result = smart_fallback(df[['name', 'email']], BlockingMode.DEDUPE, 'default')
        
        assert result['is_fallback'] == True
        # Should detect encoding issues if ExoticDataDetector is available
        if ExoticDataDetector:
            assert any(term in result['fallback_reason'].lower() for term in ['encoding', 'unicode', 'best available'])


class TestEndToEndExoticScenarios:
    """End-to-end tests with realistic exotic data scenarios."""
    
    def test_multilingual_customer_matching(self):
        """Test matching customers with names in different scripts."""
        # Source data (Latin script)
        source_df = pd.DataFrame({
            'customer_id': ['C001', 'C002', 'C003'],
            'name': ['Mohamed Ahmed', 'Sergey Ivanov', 'Takeshi Yamamoto'],
            'email': ['mohamed@example.com', 'sergey@example.ru', 'takeshi@example.jp']
        })
        
        # Reference data (native scripts)
        ref_df = pd.DataFrame({
            'id': ['R001', 'R002', 'R003'],
            'name': ['محمد أحمد', 'Сергей Иванов', '山本武'],
            'email': ['mohamed@example.com', 'sergey@example.ru', 'takeshi@example.jp']
        })
        
        mappings = [
            {'source': 'name', 'ref': 'name', 'weight': 0.7},
            {'source': 'email', 'ref': 'email', 'weight': 1.0}
        ]
        
        # The system should handle the script mismatch gracefully
        result = determine_optimal_blocking_strategy(
            df_src=source_df,
            ref_df=ref_df,
            mappings=mappings,
            mode=BlockingMode.MATCH,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        # Should probably block on email since names are in different scripts
        log.info(f"Multilingual matching strategy: {result}")
    
    def test_social_media_deduplication(self):
        """Test deduplication of social media profiles with emoji."""
        df = pd.DataFrame({
            'user_id': ['U001', 'U002', 'U003', 'U004', 'U005'],
            'username': ['CoolDude😎', 'cooldude', 'COOLDUDE😎', 'cool_dude', 'CoolDude'],
            'email': ['cool@example.com', 'cool@example.com', None, 'cooldude@example.com', 'cool@example.com'],
            'display_name': ['John 😎', 'John', 'JOHN 😎', 'John D.', 'John']
        })
        
        mappings = [
            {'source': 'username', 'weight': 1.0},
            {'source': 'email', 'weight': 0.9},
            {'source': 'display_name', 'weight': 0.7}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        assert result['apply_blocking'] == True
        log.info(f"Social media dedup strategy: {result}")
    
    def test_mixed_quality_real_world(self):
        """Test with realistic messy data."""
        df = ExoticDataGenerator.create_mixed_quality_data()
        
        mappings = [
            {'source': 'name', 'weight': 1.0},
            {'source': 'company', 'weight': 0.9},
            {'source': 'phone', 'weight': 0.8},
            {'source': 'address', 'weight': 0.7}
        ]
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default',
            weight_profile='balanced'
        )
        
        assert result['apply_blocking'] == True
        # Should handle the various formats appropriately
        log.info(f"Mixed quality data strategy: {result}")


# Performance benchmarks
class TestExoticDataPerformance:
    """Performance tests to ensure exotic data handling doesn't degrade performance."""
    
    def test_detection_performance(self):
        """Test that exotic data detection is fast enough."""
        import time
        
        # Create a large dataset
        large_df = pd.DataFrame({
            'col1': ['test'] * 10000,
            'col2': ['مرحبا'] * 10000,
            'col3': ['你好'] * 10000,
            'col4': ['Hello 😊'] * 10000
        })
        
        start_time = time.time()
        
        for col in large_df.columns:
            ExoticDataDetector.detect_characteristics(large_df[col].head(1000))
        
        elapsed = time.time() - start_time
        
        # Should complete in under 1 second for 4 columns
        assert elapsed < 1.0, f"Detection too slow: {elapsed:.3f}s"
        log.info(f"Exotic detection performance: {elapsed:.3f}s for 4 columns")
    
    def test_blocking_performance_with_exotic_data(self):
        """Test that blocking strategy selection performs well with exotic data."""
        import time
        
        # Create a mixed dataset
        df = pd.concat([
            ExoticDataGenerator.create_thai_business_data(),
            ExoticDataGenerator.create_arabic_mixed_products(),
            ExoticDataGenerator.create_cjk_companies()
        ], ignore_index=True)
        
        # Ensure we have enough data
        df = pd.concat([df] * 20, ignore_index=True)  # ~300 rows
        
        mappings = [
            {'source': col, 'weight': 1.0}
            for col in df.columns if col != 'id'
        ][:3]  # Use first 3 columns
        
        start_time = time.time()
        
        result = determine_optimal_blocking_strategy(
            df_src=df,
            ref_df=None,
            mappings=mappings,
            mode=BlockingMode.DEDUPE,
            ruleset='default'
        )
        
        elapsed = time.time() - start_time
        
        # Should complete in under 5 seconds even with exotic data
        assert elapsed < 5.0, f"Strategy selection too slow: {elapsed:.3f}s"
        assert result['apply_blocking'] == True
        log.info(f"Blocking strategy performance: {elapsed:.3f}s for exotic data")


if __name__ == "__main__":
    # Run tests with pytest
    pytest.main([__file__, '-v', '--log-cli-level=INFO'])
