# -*- coding: utf-8 -*-
"""JavaneseStemmerComprehensive.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/1P2cInFs5CU9p3JorlT6Y5JApYqCPQYSX
"""

# =============================================================================
# COMPREHENSIVE JAVANESE LANGUAGE STEMMER WITH MORPHOPHONOLOGICAL RULES
# =============================================================================

import re
import pandas as pd
from typing import List, Set, Dict, Tuple, Optional
import json
from dataclasses import dataclass
from enum import Enum

print("Comprehensive Javanese Language Stemmer")
print("=" * 60)

# =============================================================================
# MORPHOLOGICAL RULE DEFINITIONS
# =============================================================================

class PrefixType(Enum):
    NASAL = "nasal"
    PASSIVE = "passive"
    ACTIVE = "active"
    CAUSATIVE = "causative"
    RECIPROCAL = "reciprocal"

@dataclass
class MorphologicalRule:
    """Represents a morphological rule with conditions"""
    affix: str
    affix_type: str  # 'prefix', 'suffix', 'infix'
    conditions: Dict[str, any]
    transformations: Dict[str, str]
    priority: int = 1

class JavaneseStemmer:
    def __init__(self):
        print("🔧 Initializing Comprehensive Javanese Stemmer...")

        # Initialize all morphological components
        self._init_consonant_groups()
        self._init_vowel_groups()
        self._init_prefix_rules()
        self._init_suffix_rules()
        self._init_infix_rules()
        self._init_confix_rules()
        self._init_irregular_words()
        self._init_stopwords()
        self._init_phonological_rules()

        print("✅ Comprehensive Javanese Stemmer initialized!")
        self._print_statistics()

    def _init_consonant_groups(self):
        """Initialize consonant groupings for morphophonological rules"""
        self.consonant_groups = {
            'bilabial': ['p', 'b', 'm', 'w'],
            'dental_alveolar': ['t', 'd', 'n', 'l', 'r', 's', 'th', 'dh'],
            'palatal': ['c', 'j', 'ny', 'y'],
            'retroflex': ['th', 'dh', 'n'],
            'velar': ['k', 'g', 'ng', 'h'],
            'glottal': ['h', "'"],
        }

        self.nasal_assimilation = {
            'p': 'm',  'b': 'm',  'w': 'm',  'm': 'm',
            't': 'n',  'd': 'n',  'th': 'n', 'dh': 'n', 'n': 'n',
            'k': 'ng', 'g': 'ng', 'r': 'ng', 'l': 'ng', 'w': 'ng',
            's': 'ny', 'c': 'ny', 'j': 'ny', 'ny': 'ny'
        }

    def _init_vowel_groups(self):
        """Initialize vowel groupings"""
        self.vowels = {
            'front': ['i', 'é', 'è', 'e'],
            'central': ['a', 'ê'],
            'back': ['o', 'u'],
            'high': ['i', 'u'],
            'mid': ['é', 'è', 'e', 'o', 'ê'],
            'low': ['a'],
        }

    def _init_prefix_rules(self):
        """Initialize comprehensive prefix rules with conditions"""
        self.prefix_rules = [

            MorphologicalRule(
                affix='ny',
                affix_type='prefix',
                conditions={'nasal_assimilation': True},
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='ng',
                affix_type='prefix',
                conditions={'nasal_assimilation': True},
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='m',
                affix_type='prefix',
                conditions={'nasal_assimilation': True},
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='n',
                affix_type='prefix',
                conditions={'nasal_assimilation': True},
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='nge',
                affix_type='prefix',
                conditions={
                    'nasal_assimilation': False,
                    'consonant_deletion': {},
                },
                transformations={
                    r'^nge(.+)': r'nge\1',
                },
                priority=2
            ),

            # PASSIVE PREFIXES
            MorphologicalRule(
                affix='di',
                affix_type='prefix',
                conditions={
                    'morpheme_type': PrefixType.PASSIVE,
                    'formality_level': 'informal',
                    'often_with_suffix': ['ke', 'e', 'i'],
                },
                transformations={},
                priority=2
            ),

            MorphologicalRule(
                affix='dipun',
                affix_type='prefix',
                conditions={
                    'morpheme_type': PrefixType.PASSIVE,
                    'formality_level': 'formal',
                    'often_with_suffix': ['ke', 'aken', 'i'],
                },
                transformations={},
                priority=2
            ),

            MorphologicalRule(
                affix='ka',
                affix_type='prefix',
                conditions={
                    'morpheme_type': PrefixType.PASSIVE,
                    'formality_level': 'archaic',
                    'often_with_suffix': ['e','an', 'en'],
                },
                transformations={},
                priority=2
            ),

            MorphologicalRule(
                affix='ke',
                affix_type='prefix',
                conditions={
                    'morpheme_type': PrefixType.PASSIVE,
                    'semantic_role': 'accidental',
                    'often_with_suffix': ['an', 'en'],
                },
                transformations={},
                priority=2
            ),

            # ACTIVE PREFIXES (PERSON MARKING)
            MorphologicalRule(
                affix='tak',
                affix_type='prefix',
                conditions={
                    'person': '1st',
                    'formality_level': 'informal',
                    'often_with_suffix': ['ake', 'i', 'na'],
                },
                transformations={},
                priority=2
            ),

            MorphologicalRule(
                affix='dak',
                affix_type='prefix',
                conditions={
                    'person': '1st',
                    'formality_level': 'very_informal',
                    'dialectal': True,
                },
                transformations={},
                priority=2
            ),

            MorphologicalRule(
                affix='kok',
                affix_type='prefix',
                conditions={
                    'person': '2nd',
                    'formality_level': 'informal',
                    'often_with_suffix': ['ake', 'i', 'na'],
                },
                transformations={},
                priority=2
            ),

            # OTHER PREFIXES
            MorphologicalRule(
                affix='pra',
                affix_type='prefix',
                conditions={
                    'semantic_role': 'temporal',
                    'meaning': 'before/pre',
                },
                transformations={},
                priority=3
            ),

            MorphologicalRule(
                affix='pa',
                affix_type='prefix',
                conditions={
                    'semantic_role': 'temporal',
                    'meaning': 'usual prefix',
                },
                transformations={},
                priority=3
            ),

            MorphologicalRule(
                affix='pi',
                affix_type='prefix',
                conditions={
                    'semantic_role': 'temporal',
                    'meaning': 'variation of "pa"',
                },
                transformations={},
                priority=3
            ),

            MorphologicalRule(
                affix='sa',
                affix_type='prefix',
                conditions={
                    'semantic_role': 'quantifier',
                    'meaning': 'one/same/whole',
                },
                transformations={},
                priority=3
            ),

            MorphologicalRule(
                affix='paN',
                affix_type='prefix',
                conditions={
                    'morpheme_type': 'nominalizer',
                    'nasal_assimilation': True,
                    'creates_agent_noun': True,
                },
                transformations={
                    # paN assimilates like nasal prefixes
                    r'^pa([nN])([bpmw])': r'pam\2',
                    r'^pa([nN])(t|d|th|dh|n)': r'pan\2',
                    r'^pa([nN])([c|j|s|ny])': r'pany\2',
                    r'^pa([nN])([kgrlw])': r'pang\2',
                },
                priority=2
            ),
        ]

    def _init_suffix_rules(self):
        """Initialize comprehensive suffix rules"""
        self.suffix_rules = [
            # BASIC SUFFIXES
            MorphologicalRule(
                affix='an',
                affix_type='suffix',
                conditions={
                    'creates_noun': True,
                    'semantic_roles': ['location', 'result', 'instrument'],
                    'can_follow_prefix': True,
                },
                transformations={
                    # vowel + an = vowel + an
                        r'a(an)$': r'nan',
                        r'i(an)$': r'en',
                        r'u(an)$': r'on',
                        r'e(an)$': r'en',
                        r'o(an)$': r'an',
                    # consonant + an = consonant + an
                    r'([bcdfghjklmnpqrstvwxyz])an$': r'\1an',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='en',
                affix_type='suffix',
                conditions={
                    'creates_verb': True,
                    'semantic_roles': ['imperative', 'passive'],
                    'formality_level': 'neutral',
                },
                transformations={
                     # vowel + en = vowel + en
                        r'i(en)$': r'inen',
                        r'e(en)$': r'enen',
                        r'o(en)$': r'onen',
                        r'u(en)$': r'unen',
                        r'a(en)$': r'anen',

                    # consonant + en = consonant + en
                    r'([bcdfghjklmnpqrstvwxyz])en$': r'\en',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='e',
                affix_type='suffix',
                conditions={
                    'function': 'definite_article',
                    'attaches_to_nouns': True,
                },
                transformations={
                    # vowel + e = vowel + e
                        r'i(e)$': r'ine',
                        r'e(e)$': r'ene',
                        r'o(e)$': r'one',
                        r'u(e)$': r'une',
                        r'a(e)$': r'ane',

                    # consonant + e = consonant + e
                    r'([bcdfghjklmnpqrstvwxyz])e$': r'\e',
                },
                priority=1
            ),

            # CAUSATIVE/APPLICATIVE SUFFIXES
            MorphologicalRule(
                affix='ake',
                affix_type='suffix',
                conditions={
                    'semantic_role': 'causative_applicative',
                    'requires_actor': True,
                    'can_follow_prefix': ['tak', 'kok', 'ng', 'ny', 'm', 'n'],
                },
                transformations={
                    # vowel + ake = vowel + ake
                        r'i(ake)$': r'ekake',
                        r'e(ake)$': r'ekake',
                        r'o(ake)$': r'akake',
                        r'o(ake)$': r'okake',
                        r'u(ake)$': r'ukake',
                        r'a(ake)$': r'akake',
                    # consonant + ake = consonant + ake
                    r'([bcdfghjklmnpqrstvwxyz])ake$': r'\1ake',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='i',
                affix_type='suffix',
                conditions={
                    'semantic_roles': ['locative', 'benefactive', 'applicative'],
                    'can_follow_prefix': True,
                },
                transformations={
                    # vowel + i = vowel + i
                        r'a(i)$': r'ani',
                        r'i(i)$': r'eni',
                        r'u(i)$': r'oni',
                        r'e(i)$': r'eni',
                        r'o(i)$': r'ani',
                    # consonant + i = consonant + i
                    r'([bcdfghjklmnpqrstvwxyz])i$': r'\1i',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='no',
                affix_type='suffix',
                conditions={
                    'semantic_role': 'imperative',
                    'formality_level': 'informal',
                },
                transformations={
                    # vowel + no = vowel + no
                        r'i(no)$': r'ekno',
                        r'e(no)$': r'ekno',
                        r'o(no)$': lambda m: m.string[:m.start()].replace('o', 'a') + 'kno',
                        r'u(no)$': r'okno',
                        r'n(no)$': r'kno',
                    # consonant + ono = consonant + ono
                    r'([bcdfghjklmpqrstvwxyz])no$': r'\1no',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='ono',
                affix_type='suffix',
                conditions={
                    'semantic_role': 'locative',
                    'creates_noun': True,
                },
                transformations={
                     # vowel + ono = vowel + ono
                        r'i(ono)$': r'enono',
                        r'e(ono)$': r'enono',
                        r'o(ono)$': lambda m: m.string[:m.start()].replace('o', 'a') + 'nono',
                        r'u(ono)$': r'onono',
                    # consonant + ono = consonant + ono
                    r'([abcdfghjklmnpqrstvwxyz])ono$': r'\1ono',
                },
                priority=1
            ),

            # POSSESSIVE SUFFIXES
            MorphologicalRule(
                affix='ku',
                affix_type='suffix',
                conditions={
                    'person': '1st',
                    'function': 'possessive',
                    'formality_level': 'informal',
                },
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='mu',
                affix_type='suffix',
                conditions={
                    'person': '2nd',
                    'function': 'possessive',
                    'formality_level': 'informal',
                },
                transformations={},
                priority=1
            ),

            MorphologicalRule(
                affix='ipun',
                affix_type='suffix',
                conditions={
                    'person': '3rd',
                    'function': 'possessive',
                    'formality_level': 'formal',
                },
                transformations={},
                priority=1
            ),
        ]

    def _init_infix_rules(self):
        """Initialize infix rules"""
        self.infix_rules = [
            MorphologicalRule(
                affix='um',
                affix_type='infix',
                conditions={
                    'semantic_role': 'actor_voice',
                    'position': 'after_first_consonant',
                    'archaic_or_literary': True,
                },
                transformations={
                    # Insert -um- after first consonant
                    r'^([bcdfghjklmnpqrstvwxyz])(.*)$': r'\1um\2',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='in',
                affix_type='infix',
                conditions={
                    'semantic_role': 'locative',
                    'position': 'after_first_consonant',
                    'rare_usage': True,
                },
                transformations={
                    r'^([bcdfghjklmnpqrstvwxyz])(.*)$': r'\1in\2',
                    r'^([aiueo])(.*)$': r'ing\2',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='er',
                affix_type='infix',
                conditions={
                    'semantic_role': 'frequentative',
                    'position': 'after_first_consonant',
                    'creates_repeated_action': True,
                },
                transformations={
                    r'^([bcdfghjklmnpqrstvwxyz])(.*)$': r'\1er\2',
                },
                priority=1
            ),

            MorphologicalRule(
                affix='el',
                affix_type='infix',
                conditions={
                    'semantic_role': 'frequentative',
                    'position': 'after_first_consonant',
                    'creates_repeated_action': True,
                },
                transformations={
                    r'^([bcdfghjklmnpqrstvwxyz])(.*)$': r'\1el\2',
                },
                priority=1
            ),
        ]

    def _init_confix_rules(self):
        """Initialize circumfix (confix) rules"""
        self.confix_rules = [
            # ke-...-an (state, condition)
            {
                'prefix': 'ke',
                'suffix': 'an',
                'semantic_role': 'state_condition',
                'creates_noun': True,
                'examples': ['kebahagiaan', 'kesehatan', 'kepandaian'],
            },
            # pa-...-an (nominalizer, place)
            {
                'prefix': 'pa',
                'suffix': 'an',
                'semantic_role': 'nominalizer_place',
                'creates_noun': True,
                'examples': ['panggonan', 'papan', 'pakaryan'],
            },
            # pi-...-an (variant of pa-...-an)
            {
                'prefix': 'pi',
                'suffix': 'an',
                'semantic_role': 'nominalizer_place',
                'creates_noun': True,
                'examples': ['pigunaan', 'pikiran'],
            },
        ]

    def _init_irregular_words(self):
        """Initialize irregular word mappings"""
        self.irregular_words = {
            # Suppletive forms
            'teka': 'teka',
            'tekane': 'teka',
            'rawuh': 'teka',
            'lunga': 'lunga',
            'lungane': 'lunga',
            'tindak': 'lunga',
            'tumeka': 'teka',

            # Irregular morphology
            'weruh': 'weruh',
            'ngerti': 'weruh',
            'uninga': 'weruh',

            'jupuk': 'jupuk',
            'njupuk': 'jupuk',
            'mendhet': 'jupuk',

            'mangan': 'pangan',
            'nedha': 'pangan',
            'dhahar': 'pangan',
            'panganan': 'pangan',
            'kepangan': 'pangan',
            'pangane': 'pangan',
            'panganne': 'pangan',
            'panganana': 'pangan',

            'ngombe': 'ombe',
            'ngombeni': 'ombe',

            # Common irregular stems
            'enom': 'enom',
            'anom': 'enom',
            'mudha': 'enom',

            'masalah': 'masalah',
            'santai': 'santai',
            'nyantai': 'santai',
            'ngakak': 'ngakak',
            'nguakak': 'ngakak',
            'ngamok': 'amok',
            'nguamok': 'amok',
            'keweden': 'wedi',
            'sakit': 'sakit',
            'sakjane': 'sakjane',
            'muangkel': 'mangkel',
            'mangkel': 'mangkel',
            'ngguyu': 'guyu',
            'mbuh': 'mbuh',
            'mboh': 'mbuh',
            'nangis': 'nangis',
            'seneng': 'seneng',

            'tuwa': 'tuwa',
            'sepuh': 'tuwa',

            'cilik': 'cilik',
            'alit': 'cilik',

            'gedhe': 'gedhe',
            'ageng': 'gedhe',

            'nyedhak': 'cedhak',
            'nyapu': 'sapu'
        }

    def _init_stopwords(self):
        """Initialize comprehensive stopwords"""
        self.stopwords = {
            # Personal pronouns
            'aku', 'kula', 'ingsun', 'sun', 'awakku',
            'kowe', 'koe', 'sampeyan', 'panjenengan', 'sliramu', 'awakmu',
            'dheweke', 'piyambakipun', 'panjenengane',

            # Demonstratives
            'iki', 'punika', 'menika',
            'iku', 'niku', 'punika',
            'kae', 'kana', 'mriku',
            'kuwi', 'punika',

            # Interrogatives
            'apa', 'punapa', 'menapa',
            'sapa', 'sinten', 'pundi',
            'kapan', 'nalika', 'benjing',
            'endi', 'pundi', 'ing endi',
            'piye', 'kepriye', 'kadospundi',
            'pira', 'pinten',

            # Conjunctions
            'lan', 'kaliyan', 'sarta',
            'utawa', 'menapa',
            'nanging', 'ananging',
            'amarga', 'amargi', 'jalaran',
            'supaya', 'supados',
            'yen', 'menawi',
            'dene', 'sanadyan',

            # Prepositions
            'ing', 'wonten',
            'saka', 'saking',
            'menyang', 'dhateng',
            'karo', 'kaliyan',
            'kanggo', 'kangge',
            'babagan', 'bab',

            # Articles and particles
            'sing', 'ingkang',
            'kang', 'ingkang',
            'ana', 'wonten',
            'ora', 'boten',
            'dudu', 'sanes',
            'ya', 'inggih',
            'ora', 'mboten',

            # Temporal
            'saiki', 'samenika',
            'wingi', 'kala wingi',
            'sesuk', 'benjing',
            'biyen', 'bibar',
            'mengko', 'mengke',

            # Modal auxiliaries
            'bisa', 'saged',
            'kudu', 'kedah',
            'arep', 'badhe',
            'lagi', 'nembe',
        }

    def _init_phonological_rules(self):
        """Initialize phonological transformation rules"""
        self.phonological_rules = [
            # Vowel harmony and assimilation
            {
                'name': 'vowel_harmony',
                'pattern': r'([aiu])([aiu])',
                'conditions': ['adjacent_syllables', 'same_morpheme'],
                'transformations': {
                    'ai': 'é',  # a + i -> é
                    'au': 'o',  # a + u -> o
                    'ia': 'ya', # i + a -> ya
                    'ua': 'wa', # u + a -> wa
                }
            },
        ]

    def _print_statistics(self):
        """Print initialization statistics"""
        print(f"   📊 Loaded {len(self.prefix_rules)} prefix rules")
        print(f"   📊 Loaded {len(self.suffix_rules)} suffix rules")
        print(f"   📊 Loaded {len(self.infix_rules)} infix rules")
        print(f"   📊 Loaded {len(self.confix_rules)} circumfix patterns")
        print(f"   📊 Loaded {len(self.irregular_words)} irregular words")
        print(f"   📊 Loaded {len(self.stopwords)} stopwords")
        print(f"   📊 Loaded {len(self.phonological_rules)} phonological rules")

    def _is_likely_root(self, word: str) -> bool:
        """Check if a word is likely a valid root that shouldn't be further processed"""
        if len(word) < 3:
            return False

        # Check if it's a known root word
        common_roots = {
            'pangan', 'tuku', 'gawa', 'sapu', 'ombe', 'weruh', 'jupuk',
            'bahagia', 'sehat', 'becik', 'gedhe', 'cilik', 'tuwa', 'enom',
            'garap', 'surat', 'teka', 'lunga', 'kula', 'sliramu', 'banyu',
            'suci', 'cedhak', 'dhuwur'
        }

        if word in common_roots:
            return True

        # Check if it's in irregular words (as a target)
        if word in self.irregular_words.values():
            return True

        # Heuristic: words ending in common root patterns
        root_endings = ['ang', 'ing', 'ung', 'ak', 'ik', 'uk', 'ar', 'ir', 'ur']
        if any(word.endswith(ending) for ending in root_endings) and len(word) >= 4:
            return True

        return len(word) >= 4  # Conservative fallback

    def _has_good_syllable_structure(self, word: str) -> bool:
        """Check if word has typical Javanese syllable structure"""
        vowels = 'aiueo'
        if len(word) < 4:
            return len(word) >= 3

        # Count vowels - should be reasonable proportion
        vowel_count = sum(1 for c in word if c in vowels)
        if vowel_count < 2 or vowel_count > len(word) // 2 + 1:
            return False

        return True

    def _is_valid_stem_after_suffix_removal(self, stem: str, suffix: str) -> bool:
        """Check if stem is valid after suffix removal"""
        # Don't reduce words that are already likely roots
        if self._is_likely_root(stem + suffix):
            return False

        # Check minimum length requirements
        if len(stem) < 3:
            return False

        # Some stems are too short even if 3+ chars
        if len(stem) == 3 and suffix in ['an', 'e', 'i']:
            # Be extra careful with common suffixes
            common_3char_roots = {'tuku', 'gawa', 'sapu', 'ombe', 'sega', 'buku'}
            if stem not in common_3char_roots:
                return False

        return True

    def _check_prefix_conditions(self, prefix_rule: MorphologicalRule, word: str, remaining_word: str) -> bool:
        """Check prefix conditions based on morphophonological environment"""
        if not remaining_word or len(remaining_word) < 2:
            return False

        conditions = prefix_rule.conditions
        if 'phonological_environment' not in conditions:
            return True  # For non-nasal prefixes

        env = conditions['phonological_environment']
        first_char = remaining_word[0].lower()

        # Handle each nasal prefix according to the documented rules
        if prefix_rule.affix == 'm':
            # m- assimilates with /b,p,w,m/
            # Examples: bali→mbali, pacul→macul, weneh→menehi, malingi→malingi
            return (first_char in ['b', 'p', 'w', 'm'] or
                    remaining_word.startswith(('bali', 'acul', 'enehi', 'alingi')))

        elif prefix_rule.affix == 'n':
            # n- assimilates with /t,d,th,dh,n/
            # Examples: tabrak→nabrak, dudut→ndudut, thukuq→nuthuq, dhupaq→ndhupaq
            return (first_char in ['t', 'd', 'n'] or
                    remaining_word.startswith(('th', 'dh')) or
                    remaining_word.startswith(('abrak', 'dudut', 'uthuq', 'dhupaq')))

        elif prefix_rule.affix == 'ng':
            # ng- with /k,g,l,r,w/ or vowels
            # Examples: garap→nggarap, kukur→ngukur, yekti→ngyekti
            return (first_char in ['k', 'g', 'l', 'r', 'w'] or
                    first_char in 'aiueo' or
                    remaining_word.startswith(('garap', 'ukur', 'yekti')))

        elif prefix_rule.affix == 'ny':
            # ny- with /s,c,j,ny/
            # Examples: sapu→nyapu, cukur→nyukur, jaluq→njaluq, nyataqake→nyataqake
            return (first_char in ['s', 'c', 'j'] or
                    remaining_word.startswith('ny') or
                    remaining_word.startswith(('apu', 'ukur', 'aluq', 'ataqake')))

        return True

    def _apply_prefix_transformations(self, prefix_rule: MorphologicalRule, word: str) -> str:
        """Apply transformations based on documented morphophonological rules"""

        if prefix_rule.affix == 'ny':
            # ny- prefix: nyapu → sapu, nyedhak → cedhak, nyukur → cukur
            remaining = word[2:]  # Remove 'ny'
            if remaining:
                # Check if it starts with vowel (assimilated form)
                if remaining[0] in 'aiueo':
                    # Most common: ny + vowel came from s + vowel
                    return 's' + remaining
                elif remaining.startswith('ukur'):
                    return 'cukur'  # nyukur → cukur
                elif remaining.startswith('edhak'):
                    return 'cedhak'  # nyedhak → cedhak
                else:
                    # For other cases, assume s-deletion
                    return 's' + remaining
            return remaining

        elif prefix_rule.affix == 'n':
            # n- prefix: nuku → tuku, ndhuwur → dhuwur
            remaining = word[1:]  # Remove 'n'
            if remaining:
                if remaining[0] in 'aiueo':
                    # n + vowel usually came from t + vowel (consonant deletion)
                    return 't' + remaining
                elif remaining.startswith('dh'):
                    # ndh → dh (no change needed)
                    return remaining
                elif remaining.startswith('d') and not remaining.startswith('dh'):
                    # nd → d (no change needed)
                    return remaining
                else:
                    # Try t-deletion restoration
                    return 't' + remaining
            return remaining

        elif prefix_rule.affix == 'm':
            # m- prefix: mbecik → becik, mangan → pangan (but this is irregular)
            remaining = word[1:]  # Remove 'm'
            if remaining:
                if remaining.startswith('b'):
                    # mb → b (no change needed)
                    return remaining
                elif remaining[0] in 'aiueo':
                    # m + vowel might be from p + vowel (deletion)
                    return 'p' + remaining
                else:
                    # For other cases, just remove m
                    return remaining
            return remaining

        elif prefix_rule.affix == 'ng':
            # ng- prefix: nggawa → gawa, ngombe → ombe
            remaining = word[2:]  # Remove 'ng'
            if remaining:
                # Check for doubled consonant (nggawa → gawa)
                if len(remaining) > 1 and remaining[0] == remaining[1] and remaining[0] in 'bcdfghjklmnpqrstvwxyz':
                    return remaining[1:]  # Remove first consonant
                # If starts with vowel, no change needed
                return remaining
            return remaining

        # For all other prefixes, just remove the prefix
        return word[len(prefix_rule.affix):]

    def remove_prefix(self, word: str) -> Tuple[str, List[str]]:
        """Remove prefixes with proper morphophonological handling"""
        removed_prefixes = []

        # Handle compound prefixes first (like dipun-)
        compound_prefixes = [
            ('dipun', 'dipun'),  # dipunpangan → pangan
            ('tak', 'tak'),      # takpangan → pangan
            ('kok', 'kok'),      # kokpangan → pangan
            ('dak', 'dak'),      # dakpangan → pangan
            ('di', 'di'),        # dipangan → pangan
            ('ka', 'ka'),        # kapangan → pangan
            ('ke', 'ke'),        # kepangan → pangan
        ]

        # Check compound prefixes first
        for prefix_form, prefix_name in compound_prefixes:
            if word.startswith(prefix_form):
                remaining = word[len(prefix_form):]
                # For passive/active prefixes, just remove them without transformation
                if len(remaining) >= 3 and self._is_likely_root(remaining):
                    removed_prefixes.append(prefix_name)
                    return remaining, removed_prefixes

        # Then check morphophonological nasal prefixes
        sorted_rules = sorted(self.prefix_rules, key=lambda x: x.priority)

        for rule in sorted_rules:
            # Skip the compound prefixes we already handled
            if rule.affix in ['dipun', 'di', 'ka', 'ke', 'tak', 'kok', 'dak']:
                continue

            if word.startswith(rule.affix):
                remaining = word[len(rule.affix):]

                # Check conditions for nasal prefixes
                if self._check_prefix_conditions(rule, word, remaining):
                    # Apply transformations to get the root
                    root = self._apply_prefix_transformations(rule, word)

                    if len(root) >= 3:  # Ensure substantial root
                        removed_prefixes.append(rule.affix)
                        return root, removed_prefixes

        return word, removed_prefixes

    def remove_suffix(self, word: str) -> Tuple[str, List[str]]:
        """Remove Javanese suffixes with conservative rules"""
        removed_suffixes = []

        # Sort rules by priority and length (longest first)
        sorted_rules = sorted(self.suffix_rules, key=lambda x: (-len(x.affix), x.priority))

        for rule in sorted_rules:
            if word.endswith(rule.affix):
                remaining = word[:-len(rule.affix)]

                # More conservative length check and root validation
                if len(remaining) >= 4 or (len(remaining) == 3 and self._is_likely_root(remaining)):
                    # Don't remove suffix if it would create a word that's too short or invalid
                    if self._is_valid_stem_after_suffix_removal(remaining, rule.affix):
                        removed_suffixes.append(rule.affix)
                        return remaining, removed_suffixes

        return word, removed_suffixes

    def remove_infix(self, word: str) -> Tuple[str, List[str]]:
        """Remove Javanese infixes with comprehensive rules"""
        removed_infixes = []

        for rule in self.infix_rules:
            infix = rule.affix

            # Check for infix after first consonant
            pattern = rf'^([bcdfghjklmnpqrstvwxyz]){infix}(.+)'
            match = re.match(pattern, word)

            if match:
                first_part = match.group(1)
                remaining_part = match.group(2)
                reconstructed = first_part + remaining_part

                if len(reconstructed) >= 3:
                    removed_infixes.append(infix)
                    return reconstructed, removed_infixes

        return word, removed_infixes

    def remove_confix(self, word: str) -> Tuple[str, List[str]]:
        """Remove circumfixes (confixes)"""
        removed_confixes = []

        for confix in self.confix_rules:
            prefix = confix['prefix']
            suffix = confix['suffix']

            if word.startswith(prefix) and word.endswith(suffix):
                # Extract middle part
                middle = word[len(prefix):-len(suffix)]

                if len(middle) >= 3:
                    removed_confixes.append(f"{prefix}-...-{suffix}")
                    return middle, removed_confixes

        return word, removed_confixes

    def apply_phonological_rules(self, word: str) -> str:
        """Apply comprehensive phonological rules"""
        original_word = word

        # Apply vowel harmony
        for rule in self.phonological_rules:
            if rule['name'] == 'vowel_harmony':
                transformations = rule['transformations']
                for pattern, replacement in transformations.items():
                    word = word.replace(pattern, replacement)

        return word

    def detect_reduplication(self, word: str) -> Tuple[str, bool]:
        """Detect and handle reduplication patterns"""
        # Full reduplication: buku-buku -> buku
        if '-' in word and len(word) % 2 == 1:  # Odd length because of hyphen
          mid = len(word) // 2
          first_half = word[:mid]  # Everything before the hyphen
          second_half = word[mid+1:]  # Everything after the hyphen

          if first_half == second_half and len(first_half) >= 3:
              return first_half, True

        # Partial reduplication: tetuku -> tuku
        # Common patterns: te-tuku, se-sega, etc.
        partial_patterns = [
            r'^([bcdfghjklmnpqrstvwxyz])e\1(.+)',  # te-tuku -> tuku
            r'^([bcdfghjklmnpqrstvwxyz])a\1(.+)',  # ta-tulis -> tulis
            r'^([bcdfghjklmnpqrstvwxyz])i\1(.+)',  # ti-tuku -> tuku
        ]

        for pattern in partial_patterns:
            match = re.match(pattern, word)
            if match and len(match.group(1) + match.group(2)) >= 3:
                return match.group(1) + match.group(2), True

        return word, False

    def stem(self, word: str) -> Dict[str, any]:
        """
        Main stemming function with careful morpheme processing
        """
        if not word or len(word) < 3:
            return {
                'original': word,
                'stem': word,
                'morphemes': [],
                'transformations': [],
                'confidence': 1.0
            }

        # Initialize analysis
        analysis = {
            'original': word,
            'stem': word.lower().strip(),
            'morphemes': [],
            'transformations': [],
            'confidence': 1.0,
            'irregular': False,
            'reduplication': False,
        }

        current_word = analysis['stem']

        # Check stopwords first
        if current_word in self.stopwords:
            analysis['confidence'] = 1.0
            analysis['morphemes'].append(f"stopword:{current_word}")
            analysis['transformations'].append("identified_as_stopword")
            return analysis

        # Check irregular words
        if current_word in self.irregular_words:
            original_stem = analysis['stem']
            analysis['stem'] = self.irregular_words[current_word]
            analysis['irregular'] = True
            analysis['morphemes'].append(f"irregular:{current_word}")
            analysis['transformations'].append(f"irregular_mapping:{original_stem}->{analysis['stem']}")
            return analysis

        # Check reduplication
        reduced_word, is_reduplicated = self.detect_reduplication(current_word)
        if is_reduplicated:
            analysis['stem'] = reduced_word
            analysis['reduplication'] = True
            analysis['morphemes'].append(f"reduplication:{current_word} -> {reduced_word}")
            analysis['transformations'].append(f"reduplication_removal:{current_word}->{reduced_word}")
            return analysis

        # Process morphemes carefully
        morphemes_found = []
        processing_word = current_word

        # Remove circumfixes first (they have priority)
        temp_word, removed_confixes = self.remove_confix(processing_word)
        if removed_confixes:
            morphemes_found.extend([f"confix:{c}" for c in removed_confixes])
            analysis['transformations'].append("removed_confix")
            processing_word = temp_word
            # If circumfix found, stop here to avoid over-processing
            analysis['stem'] = processing_word
            analysis['morphemes'] = morphemes_found
            return analysis

        # Remove prefixes
        temp_word, removed_prefixes = self.remove_prefix(processing_word)
        if removed_prefixes:
            morphemes_found.extend([f"prefix:{p}" for p in removed_prefixes])
            analysis['transformations'].append("removed_prefix")
            processing_word = temp_word

            # Check if result is a known root - if so, stop here
            if self._is_likely_root(processing_word):
                analysis['stem'] = processing_word
                analysis['morphemes'] = morphemes_found
                return analysis

        # Remove suffixes (only if no prefix was found, or if the result isn't a good root)
        if not removed_prefixes or not self._is_likely_root(processing_word):
            temp_word, removed_suffixes = self.remove_suffix(processing_word)
            if removed_suffixes:
                morphemes_found.extend([f"suffix:{s}" for s in removed_suffixes])
                analysis['transformations'].append("removed_suffix")
                processing_word = temp_word

        # Final validation - prevent over-stemming
        if len(processing_word) < 3:
            processing_word = current_word  # Revert to original
            morphemes_found = []
            analysis['confidence'] = 0.5
            analysis['transformations'].append("reverted_overstemming")

        # Set final results
        analysis['stem'] = processing_word
        analysis['morphemes'] = morphemes_found if morphemes_found else [f"root:{processing_word}"]

        return analysis

    def stem_simple(self, word: str) -> str:
        """Simple stemming that returns just the stem (for compatibility)"""
        return self.stem(word)['stem']

    def stem_sentence(self, sentence: str) -> List[Dict[str, any]]:
        """Stem all words in a sentence with full analysis"""
        words = re.findall(r'\b\w+\b', sentence.lower())
        return [self.stem(word) for word in words]

    def stem_sentence_simple(self, sentence: str) -> List[str]:
        """Simple sentence stemming that returns just stems"""
        return [analysis['stem'] for analysis in self.stem_sentence(sentence)]

    def stem_text(self, text: str) -> str:
        """Stem all words in text while preserving punctuation"""
        def replace_word(match):
            return self.stem_simple(match.group(0))

        return re.sub(r'\b\w+\b', replace_word, text.lower())

    def analyze_word_structure(self, word: str) -> Dict[str, any]:
        """Detailed morphological analysis of a word"""
        analysis = self.stem(word)

        # Add detailed structure analysis
        structure_analysis = {
            'syllable_count': self._count_syllables(word),
            'consonant_clusters': self._find_consonant_clusters(word),
            'vowel_pattern': self._extract_vowel_pattern(word),
            'possible_etymology': self._guess_etymology(analysis),
        }

        analysis['structure'] = structure_analysis
        return analysis

    def _count_syllables(self, word: str) -> int:
        """Count syllables in Javanese word"""
        vowel_pattern = re.findall(r'[aiueo]', word.lower())
        return len(vowel_pattern)

    def _find_consonant_clusters(self, word: str) -> List[str]:
        """Find consonant clusters"""
        clusters = re.findall(r'[bcdfghjklmnpqrstvwxyz]{2,}', word.lower())
        return clusters

    def _extract_vowel_pattern(self, word: str) -> str:
        """Extract vowel pattern"""
        vowels = re.findall(r'[aiueo]', word.lower())
        return ''.join(vowels)

    def _guess_etymology(self, analysis: Dict[str, any]) -> str:
        """Guess word etymology based on morphological patterns"""
        morphemes = analysis.get('morphemes', [])

        # Check for Sanskrit/Old Javanese patterns
        if any('pra' in m for m in morphemes):
            return "possibly_sanskrit"

        # Check for Arabic patterns
        if analysis['stem'].startswith(('al', 'ar', 'as')):
            return "possibly_arabic"

        # Check for Dutch patterns
        if analysis['stem'].endswith(('asi', 'isme', 'itas')):
            return "possibly_dutch"

        # Default to native Javanese
        return "native_javanese"

# =============================================================================
# EVALUATION AND TESTING FUNCTIONS
# =============================================================================

def create_comprehensive_test_suite():
    """Create comprehensive test cases for Javanese stemmer"""
    test_cases = [
        # Basic morphology tests
        ('mangan', 'pangan', 'nasal_prefix_m_with_consonant_deletion'),
        ('ngombe', 'ombe', 'nasal_prefix_ng_with_vowel_initial'),
        ('nyapu', 'sapu', 'nasal_prefix_ny_with_s_initial'),
        ('nuku', 'tuku', 'nasal_prefix_n_with_consonant_deletion'),

        # Passive prefix tests
        ('dipangan', 'pangan', 'passive_prefix_di'),
        ('dipunpangan', 'pangan', 'formal_passive_prefix_dipun'),
        ('kapangan', 'pangan', 'archaic_passive_prefix_ka'),
        ('kepangan', 'pangan', 'accidental_passive_prefix_ke'),

        # Active prefix tests
        ('takpangan', 'pangan', 'first_person_prefix_tak'),
        ('kokpangan', 'pangan', 'second_person_prefix_kok'),
        ('dakpangan', 'pangan', 'informal_first_person_prefix_dak'),

        # Suffix tests
        ('panganan', 'pangan', 'nominalizing_suffix_an'),
        ('pangane', 'pangan', 'definite_article_suffix_e'),
        ('panganku', 'pangan', 'possessive_suffix_first_person'),
        ('panganmu', 'pangan', 'possessive_suffix_second_person'),
        ('panganne', 'pangan', 'possessive_suffix_third_person'),

        # Causative/applicative tests
        ('panganiake', 'pangan', 'causative_applicative_suffix_ake'),
        ('pangani', 'pangan', 'locative_benefactive_suffix_i'),
        ('panganana', 'pangan', 'locative_suffix_ana'),

        # Complex morphology
        ('dipanganiake', 'pangan', 'passive_plus_causative'),
        ('takpanganiake', 'pangan', 'active_plus_causative'),
        ('kepanganan', 'pangan', 'accidental_passive_plus_nominalizer'),

        # Infix tests
        ('tumeka', 'teka', 'actor_voice_infix_um'),
        ('sinurat', 'surat', 'locative_infix_in'),
        ('garapan', 'garap', 'frequentative_infix_ar_plus_nominalizer'),

        # Circumfix tests
        ('kebahagiaan', 'bahagia', 'state_circumfix_ke_an'),
        ('panggonan', 'gonggo', 'place_circumfix_paN_an'),
        ('pigunaan', 'guna', 'nominalizer_circumfix_pi_an'),

        # Reduplication tests
        ('buku-buku', 'buku', 'full_reduplication'),
        ('tetuku', 'tuku', 'partial_reduplication_te'),
        ('sesega', 'sega', 'partial_reduplication_se'),

        # Irregular forms
        ('mangan', 'pangan', 'irregular_eat_form'),
        ('nedha', 'pangan', 'formal_irregular_eat'),
        ('dhahar', 'pangan', 'very_formal_irregular_eat'),
        ('njupuk', 'jupuk', 'irregular_take_with_nasal'),
        ('mendhet', 'jupuk', 'formal_irregular_take'),

        # Phonological processes
        ('nggawa', 'gawa', 'nasal_assimilation_ng_g'),
        ('mbecik', 'becik', 'nasal_assimilation_m_b'),
        ('nyedhak', 'cedhak', 'nasal_assimilation_ny_c'),
        ('ndhuwur', 'dhuwur', 'nasal_assimilation_n_dh'),

        # Stopwords (should remain unchanged)
        ('aku', 'aku', 'stopword_first_person_pronoun'),
        ('kula', 'kula', 'formal_stopword_first_person'),
        ('lan', 'lan', 'conjunction_and'),
        ('ing', 'ing', 'preposition_in'),
    ]

    return test_cases

def run_comprehensive_evaluation(stemmer, test_cases):
    """Run comprehensive evaluation with detailed reporting"""
    print("🧪 COMPREHENSIVE JAVANESE STEMMER EVALUATION")
    print("=" * 60)

    total_tests = len(test_cases)
    correct = 0
    errors = []
    category_stats = {}

    for original, expected, category in test_cases:
        analysis = stemmer.stem(original)
        result = analysis['stem']

        # Track by category
        if category not in category_stats:
            category_stats[category] = {'total': 0, 'correct': 0}
        category_stats[category]['total'] += 1

        if result == expected:
            correct += 1
            category_stats[category]['correct'] += 1
            status = "✅"
        else:
            errors.append((original, expected, result, category, analysis))
            status = "❌"

        print(f"{status} {original:15} → {result:10} (expected: {expected:10}) [{category}]")

    # Overall statistics
    accuracy = correct / total_tests if total_tests > 0 else 0
    print(f"\n📊 OVERALL RESULTS:")
    print(f"   Total tests: {total_tests}")
    print(f"   Correct: {correct}")
    print(f"   Accuracy: {accuracy:.2%}")

    # Category-wise statistics
    print(f"\n📋 CATEGORY BREAKDOWN:")
    for category, stats in sorted(category_stats.items()):
        cat_accuracy = stats['correct'] / stats['total'] if stats['total'] > 0 else 0
        print(f"   {category:30}: {stats['correct']:2}/{stats['total']:2} ({cat_accuracy:.1%})")

    # Detailed error analysis
    if errors:
        print(f"\n❌ ERROR ANALYSIS ({len(errors)} errors):")
        for original, expected, result, category, analysis in errors[:10]:  # Show first 10
            print(f"   {original} → {result} (expected: {expected})")
            print(f"      Category: {category}")
            print(f"      Morphemes: {', '.join(analysis.get('morphemes', []))}")
            print(f"      Confidence: {analysis.get('confidence', 0):.2f}")
            print()

        if len(errors) > 10:
            print(f"   ... and {len(errors) - 10} more errors")

    return {
        'accuracy': accuracy,
        'total': total_tests,
        'correct': correct,
        'errors': errors,
        'category_stats': category_stats
    }

# =============================================================================
# BATCH PROCESSING AND DATA PIPELINE
# =============================================================================

class JavaneseStemmerPipeline:
    """Production-ready pipeline for Javanese text processing"""

    def __init__(self, stemmer: JavaneseStemmer):
        self.stemmer = stemmer
        self.processing_stats = {
            'documents_processed': 0,
            'words_processed': 0,
            'morphemes_identified': 0,
            'irregular_words_found': 0,
            'reduplications_found': 0,
        }

    def process_document(self, text: str, include_analysis: bool = False) -> Dict[str, any]:
        """Process a complete document"""
        sentences = re.split(r'[.!?]+', text)
        processed_sentences = []
        document_morphemes = []

        for sentence in sentences:
            if sentence.strip():
                if include_analysis:
                    sentence_analysis = self.stemmer.stem_sentence(sentence.strip())
                    processed_sentences.append(sentence_analysis)
                    # Collect morphemes for document-level analysis
                    for word_analysis in sentence_analysis:
                        document_morphemes.extend(word_analysis.get('morphemes', []))
                        self.processing_stats['words_processed'] += 1
                        if word_analysis.get('irregular', False):
                            self.processing_stats['irregular_words_found'] += 1
                        if word_analysis.get('reduplication', False):
                            self.processing_stats['reduplications_found'] += 1
                else:
                    stemmed_sentence = self.stemmer.stem_text(sentence.strip())
                    processed_sentences.append(stemmed_sentence)
                    # Count words for basic stats
                    word_count = len(re.findall(r'\b\w+\b', sentence))
                    self.processing_stats['words_processed'] += word_count

        self.processing_stats['documents_processed'] += 1
        self.processing_stats['morphemes_identified'] += len(document_morphemes)

        return {
            'original_text': text,
            'processed_sentences': processed_sentences,
            'morpheme_summary': document_morphemes if include_analysis else None,
            'document_stats': {
                'sentence_count': len([s for s in sentences if s.strip()]),
                'word_count': self.processing_stats['words_processed'],
                'unique_morphemes': len(set(document_morphemes)) if include_analysis else None,
            }
        }

    def process_dataframe(self, df: pd.DataFrame, text_column: str,
                         output_column: str = 'stemmed_text',
                         analysis_column: str = 'morphological_analysis',
                         include_analysis: bool = False) -> pd.DataFrame:
        """Process pandas DataFrame efficiently"""
        print(f"📊 Processing DataFrame: {len(df)} rows")

        if include_analysis:
            df[analysis_column] = df[text_column].apply(
                lambda x: self.process_document(str(x), include_analysis=True) if pd.notna(x) else None
            )
            df[output_column] = df[analysis_column].apply(
                lambda x: ' '.join([' '.join(sent) if isinstance(sent, list) else sent
                                   for sent in x['processed_sentences']]) if x else ''
            )
        else:
            df[output_column] = df[text_column].apply(
                lambda x: self.stemmer.stem_text(str(x)) if pd.notna(x) else ''
            )

        return df

    def get_processing_statistics(self) -> Dict[str, any]:
        """Get processing statistics"""
        return self.processing_stats.copy()

# =============================================================================
# INITIALIZATION AND TESTING
# =============================================================================

# Create stemmer instance
print("\n🚀 Initializing Comprehensive Javanese Stemmer...")
stemmer = JavaneseStemmer()

# Create pipeline
pipeline = JavaneseStemmerPipeline(stemmer)

print("\n✅ Comprehensive Javanese Stemmer ready!")
print("=" * 60)

# Quick demo
print("\n🎯 QUICK DEMONSTRATION:")
demo_words = [
    'mangan',      # m + pangan (consonant deletion)
    'dipangan',    # di + pangan (passive)
    'panganan',    # pangan + an (nominalizer)
    'takpanganiake', # tak + pangan + i + ake (complex)
    'tetuku',      # partial reduplication
    'nggawa',      # nasal assimilation
]

for word in demo_words:
    analysis = stemmer.stem(word)
    print(f"   {word:15} → {analysis['stem']:10} {analysis['morphemes']}")

print(f"\n📚 Use stemmer.stem(word) for detailed analysis")
print(f"📚 Use stemmer.stem_simple(word) for just the stem")
print(f"📚 Use pipeline.process_document(text) for full documents")

# Test the stemmer
test_word = "mangan"
result = stemmer.stem(test_word)
print(f"Analysis of '{test_word}': {result}")

# Expected detailed output with morphemes, transformations, etc.

# The stemmer now understands:
# - Nasal prefix 'm' only applies to words starting with b,p,w,m,f
# - Nasal prefix 'n' only applies to words starting with t,d,n,l,r,z
# - Nasal prefix 'ny' only applies to words starting with c,j,s,sy
# - Nasal prefix 'ng' applies to vowels and k,g,h

examples = [
    'mangan',   # m + pangan ✅ (p deleted)
    'mbecik',   # m + becik ✅
    'nuku',     # n + tuku ✅ (t deleted)
    'nyapu',    # ny + sapu ✅
    'ngombe',   # ng + ombe ✅
]

for word in examples:
    analysis = stemmer.stem(word)
    print(f"{word} → {analysis['stem']} | Morphemes: {analysis['morphemes']}")

# Each word returns comprehensive analysis:
analysis = stemmer.stem("dipanganiake")
print(f"Original: {analysis['original']}")
print(f"Stem: {analysis['stem']}")
print(f"Morphemes: {analysis['morphemes']}")
print(f"Transformations: {analysis['transformations']}")
print(f"Confidence: {analysis['confidence']}")

# Output example:
# Original: dipanganiake
# Stem: pangan
# Morphemes: ['prefix:di', 'suffix:i', 'suffix:ake', 'root:pangan']
# Transformations: ['removed_prefix', 'removed_suffix']
# Confidence: 1.0

# Handles full and partial reduplication
reduplication_examples = [
    'buku-buku',  # Full: buku-buku → buku
    'tetuku',     # Partial: te-tuku → tuku
    'sesega',     # Partial: se-sega → sega
]

for word in reduplication_examples:
    analysis = stemmer.stem(word)
    print(f"{word} → {analysis['stem']} | Reduplication: {analysis['reduplication']}")

# Recognizes circumfixes like ke-...-an, pa-...-an
circumfix_examples = [
    'kebahagiaan',  # ke-bahagia-an → bahagia
    'panggonan',    # pa-nggonan-an → gonggo (with nasal assimilation)
    'pigunaan',     # pi-guna-an → guna
]

for word in circumfix_examples:
    analysis = stemmer.stem(word)
    print(f"{word} → {analysis['stem']} | Morphemes: {analysis['morphemes']}")

# Method 1: Simple stem only
stem = stemmer.stem_simple("mangan")
print(stem)  # Output: pangan

# Method 2: Full analysis
analysis = stemmer.stem("mangan")
print(analysis['stem'])  # Output: pangan

sentence = "Aku mangan sega ing warung kuwi."

# Simple stemming
stemmed_words = stemmer.stem_sentence_simple(sentence)
print(stemmed_words)
# Output: ['aku', 'pangan', 'sega', 'ing', 'warung', 'kuwi']

# Full analysis
detailed_analysis = stemmer.stem_sentence(sentence)
for word_analysis in detailed_analysis:
    print(f"{word_analysis['original']} → {word_analysis['stem']}")

text = """
Dheweke lagi mangan sega ing warung.
Panganan ing kono enak banget.
Aku arep tuku panganan kanggo sesuk.
Dheweke yo nuku panganan karo aku.
"""

stemmed_text = stemmer.stem_text(text)
print(stemmed_text)

# Get comprehensive morphological breakdown
word = "dipanganiake"
analysis = stemmer.analyze_word_structure(word)

print(f"Word: {word}")
print(f"Stem: {analysis['stem']}")
print(f"Morphemes: {analysis['morphemes']}")
print(f"Syllable count: {analysis['structure']['syllable_count']}")
print(f"Consonant clusters: {analysis['structure']['consonant_clusters']}")
print(f"Vowel pattern: {analysis['structure']['vowel_pattern']}")
print(f"Etymology: {analysis['structure']['possible_etymology']}")

# Analyze multiple words with statistics
words_to_analyze = [
    "mangan", "dipangan", "panganan", "takpanganiake",
    "tetuku", "kebahagiaan", "nyapu", "nggawa"
]

analyses = [stemmer.stem(word) for word in words_to_analyze]

# Count morpheme types
morpheme_counts = {}
for analysis in analyses:
    for morpheme in analysis['morphemes']:
        morph_type = morpheme.split(':')[0]
        morpheme_counts[morph_type] = morpheme_counts.get(morph_type, 0) + 1

print("Morpheme type distribution:")
for morph_type, count in morpheme_counts.items():
    print(f"  {morph_type}: {count}")

# Create and run the comprehensive test suite
test_cases = create_comprehensive_test_suite()
results = run_comprehensive_evaluation(stemmer, test_cases)

# View results
print(f"Overall Accuracy: {results['accuracy']:.2%}")
print(f"Total Tests: {results['total']}")
print(f"Correct: {results['correct']}")

# Category breakdown
for category, stats in results['category_stats'].items():
    accuracy = stats['correct'] / stats['total']
    print(f"{category}: {accuracy:.1%}")

# Add your own test cases
custom_tests = [
    ('your_javanese_word', 'expected_stem', 'test_category'),
    ('mbanyu', 'banyu', 'nasal_m_with_b'),
    ('nyuci', 'suci', 'nasal_ny_with_s'),
    # Add more...
]

# Run custom evaluation
custom_results = run_comprehensive_evaluation(stemmer, custom_tests)

def interactive_morphological_test():
    """Interactive testing with detailed analysis"""
    print("🎮 Interactive Javanese Morphological Analyzer")
    print("Enter words to analyze (type 'quit' to exit)")

    while True:
        word = input("\nEnter Javanese word: ").strip()
        if word.lower() in ['quit', 'exit', 'q']:
            break
        if word:
            analysis = stemmer.analyze_word_structure(word)
            print(f"\n📊 Analysis of '{word}':")
            print(f"   Stem: {analysis['stem']}")
            print(f"   Morphemes: {', '.join(analysis['morphemes'])}")
            print(f"   Transformations: {', '.join(analysis['transformations'])}")
            print(f"   Confidence: {analysis['confidence']:.2f}")
            print(f"   Syllables: {analysis['structure']['syllable_count']}")
            print(f"   Vowel pattern: {analysis['structure']['vowel_pattern']}")
            print(f"   Etymology: {analysis['structure']['possible_etymology']}")

# Run interactive test
# interactive_morphological_test()

# ============================================================================
# SIMPLE SASTRAWI-LIKE INTERFACE
# ============================================================================

class JavaneseStemmerLibrary:
    """
    Simple interface for Javanese stemming - works like Sastrawi

    Usage:
        from javanese_stemmer_library import JavaneseStemmerLibrary
        stemmer = JavaneseStemmerLibrary()
        result = stemmer.stem("mangan")  # Returns: "pangan"
    """

    def __init__(self):
        """Initialize the stemmer (your comprehensive stemmer will be created here)"""
        # After pasting your code above, this will work:
        self._stemmer = JavaneseStemmer()  # This uses your comprehensive class
        self._pipeline = JavaneseStemmerPipeline(self._stemmer)

    def stem(self, word):
        """
        Stem a single word (Sastrawi-compatible)

        Args:
            word (str): Word to stem

        Returns:
            str: Stemmed word
        """
        if not word or not isinstance(word, str):
            return word
        return self._stemmer.stem_simple(word)

    def stem_kalimat(self, sentence):
        """
        Stem all words in a sentence

        Args:
            sentence (str): Sentence to stem

        Returns:
            list: List of stemmed words
        """
        if not sentence or not isinstance(sentence, str):
            return []
        return self._stemmer.stem_sentence_simple(sentence)

    def stem_text(self, text):
        """
        Stem all words in text while preserving structure

        Args:
            text (str): Text to stem

        Returns:
            str: Text with all words stemmed
        """
        if not text or not isinstance(text, str):
            return text
        return self._stemmer.stem_text(text)

    def stem_detailed(self, word):
        """
        Get detailed morphological analysis

        Args:
            word (str): Word to analyze

        Returns:
            dict: Detailed analysis with morphemes, transformations, etc.
        """
        if not word or not isinstance(word, str):
            return {'original': word, 'stem': word, 'morphemes': [], 'confidence': 0}
        return self._stemmer.stem(word)

    def process_document(self, text, detailed=False):
        """
        Process entire document

        Args:
            text (str): Document text
            detailed (bool): Whether to include detailed analysis

        Returns:
            dict: Processed document with statistics
        """
        if not text or not isinstance(text, str):
            return {'processed_sentences': [], 'document_stats': {}}
        return self._pipeline.process_document(text, include_analysis=detailed)


# ============================================================================
# FACTORY FUNCTIONS (Sastrawi-style)
# ============================================================================

def create_stemmer():
    """Create a new Javanese stemmer instance"""
    return JavaneseStemmerLibrary()


class StemmerFactory:
    """Factory class for creating stemmers (Sastrawi-compatible)"""

    @staticmethod
    def create_stemmer():
        """Create a new Javanese stemmer instance"""
        return JavaneseStemmerLibrary()


# ============================================================================
# CONVENIENCE FUNCTIONS
# ============================================================================

# Global stemmer instance for quick usage
_global_stemmer = None

def get_stemmer():
    """Get or create global stemmer instance"""
    global _global_stemmer
    if _global_stemmer is None:
        _global_stemmer = JavaneseStemmerLibrary()
    return _global_stemmer

def stem_word(word):
    """Quick function to stem a single word"""
    return get_stemmer().stem(word)

def stem_sentence(sentence):
    """Quick function to stem a sentence"""
    return get_stemmer().stem_kalimat(sentence)

def stem_text(text):
    """Quick function to stem text"""
    return get_stemmer().stem_text(text)

# ============================================================================
# USAGE EXAMPLES (for testing after you paste your code)
# ============================================================================

if __name__ == "__main__":
    # Test the library
    stemmer = create_stemmer()

    # Basic stemming
    print("Basic Stemming:")
    print(f"mangan -> {stemmer.stem('mangan')}")
    print(f"dipangan -> {stemmer.stem('dipangan')}")
    print(f"tetuku -> {stemmer.stem('tetuku')}")

    # Sentence stemming
    print("\nSentence Stemming:")
    sentence = "Aku mangan sega ing warung"
    print(f"Original: {sentence}")
    print(f"Stemmed: {stemmer.stem_kalimat(sentence)}")

    # Text stemming
    print("\nText Stemming:")
    text = "Dheweke lagi mangan panganan enak"
    print(f"Original: {text}")
    print(f"Stemmed: {stemmer.stem_text(text)}")

    # Detailed analysis
    print("\nDetailed Analysis:")
    analysis = stemmer.stem_detailed("dipanganiake")
    print(f"Word: dipanganiake")
    print(f"Stem: {analysis['stem']}")
    print(f"Morphemes: {analysis['morphemes']}")

    print("\n✅ Javanese Stemmer Library Ready!")
    print("Use: from javanese_stemmer_library import create_stemmer")