"""
MikoshiLang Extended Functions - Set 16
Natural language processing, computational linguistics, and text analysis
"""
import sympy as sp
from sympy import symbols, log, sqrt

def get_rules():
    """Extended set 16: NLP and linguistics (200 functions)"""
    x = symbols('x')
    
    return [
        # ===== NLP BASICS (50 functions) =====
        ('Tokenize[text_]', lambda t: str(t).split()),
        ('SentenceTokenize[text_]', lambda t: str(t).split('. ')),
        ('WordTokenize[text_]', lambda t: str(t).split()),
        ('CharacterTokenize[text_]', lambda t: list(str(t))),
        ('NGramTokenize[text_, n_]', lambda t, n: [str(t)[i:i+int(n)] for i in range(len(str(t))-int(n)+1)]),
        ('RemovePunctuation[text_]', lambda t: ''.join(c for c in str(t) if c.isalnum() or c.isspace())),
        ('RemoveStopWords[tokens_, language_]', lambda tok, lang: [w for w in tok if w.lower() not in ['the', 'a', 'an']]),
        ('Lowercase[text_]', lambda t: str(t).lower()),
        ('Uppercase[text_]', lambda t: str(t).upper()),
        ('TitleCase[text_]', lambda t: str(t).title()),
        ('StemWords[words_]', lambda w: [word[:-3] if len(word) > 3 else word for word in w]),
        ('PorterStemmer[word_]', lambda w: str(w)[:-3] if len(str(w)) > 3 else str(w)),
        ('LancasterStemmer[word_]', lambda w: str(w)[:-2] if len(str(w)) > 2 else str(w)),
        ('SnowballStemmer[word_, language_]', lambda w, l: str(w)[:-3] if len(str(w)) > 3 else str(w)),
        ('Lemmatize[word_, pos_]', lambda w, p: str(w)),
        ('WordNetLemmatizer[word_]', lambda w: str(w)),
        ('POSTag[tokens_]', lambda t: [(w, 'NN') for w in t]),
        ('PartOfSpeechTag[text_]', lambda t: [(w, 'NN') for w in str(t).split()]),
        ('NamedEntityRecognition[text_]', lambda t: [('Entity', 'ORG')]),
        ('ChunkParsing[tagged_]', lambda t: [['NP', t]]),
        ('DependencyParsing[text_]', lambda t: [('word', 'ROOT', 0)]),
        ('ConstituencyParsing[text_]', lambda t: '(S (NP) (VP))'),
        ('SyntaxTree[text_]', lambda t: '(S (NP) (VP))'),
        ('SemanticRole[text_]', lambda t: [('verb', 'agent', 'patient')]),
        ('CoreferenceResolution[text_]', lambda t: [(0, 5)]),
        ('SentimentAnalysis[text_]', lambda t: 0.5),
        ('SentimentPolarity[text_]', lambda t: 0.5),
        ('SentimentSubjectivity[text_]', lambda t: 0.6),
        ('EmotionDetection[text_]', lambda t: {'joy': 0.3, 'sadness': 0.1}),
        ('AspectBasedSentiment[text_]', lambda t: {'aspect1': 0.7}),
        ('OpinionMining[text_]', lambda t: [('feature', 'positive')]),
        ('IntentClassification[text_]', lambda t: 'query'),
        ('TopicModeling[documents_, n_]', lambda d, n: [[0.5, 0.3, 0.2]]),
        ('LDATopicModel[documents_, topics_]', lambda d, t: [[0.5]]),
        ('LSATopicModel[documents_, topics_]', lambda d, t: [[0.5]]),
        ('NMFTopicModel[documents_, topics_]', lambda d, t: [[0.5]]),
        ('Word2Vec[corpus_, dimensions_]', lambda c, d: [[0.1]*int(d)]),
        ('GloVe[corpus_, dimensions_]', lambda c, d: [[0.1]*int(d)]),
        ('FastText[corpus_, dimensions_]', lambda c, d: [[0.1]*int(d)]),
        ('BERTEmbedding[text_]', lambda t: [[0.1]*768]),
        ('ELMoEmbedding[text_]', lambda t: [[0.1]*1024]),
        ('SentenceEmbedding[text_]', lambda t: [0.1]*512),
        ('DocumentEmbedding[text_]', lambda t: [0.1]*300),
        ('SemanticSimilarity[text1_, text2_]', lambda t1, t2: 0.8),
        ('TextualEntailment[premise_, hypothesis_]', lambda p, h: True),
        ('ParaphraseDetection[text1_, text2_]', lambda t1, t2: True),
        ('MachineTranslation[text_, source_, target_]', lambda t, s, tgt: str(t)),
        ('TextSummarization[text_, ratio_]', lambda t, r: str(t)[:int(len(str(t))*r)]),
        ('AbstractiveSummarization[text_]', lambda t: str(t)[:50]),
        ('ExtractiveSummarization[text_, n_]', lambda t, n: '. '.join(str(t).split('. ')[:int(n)])),
        
        # ===== TEXT METRICS (40 functions) =====
        ('LexicalDiversity[text_]', lambda t: len(set(str(t).split())) / len(str(t).split())),
        ('TypeTokenRatio[text_]', lambda t: len(set(str(t).split())) / len(str(t).split())),
        ('HapaxLegomena[text_]', lambda t: [w for w in set(str(t).split()) if str(t).split().count(w) == 1]),
        ('HapaxDislegomena[text_]', lambda t: [w for w in set(str(t).split()) if str(t).split().count(w) == 2]),
        ('YuleK[text_]', lambda t: 10000 * (sum(v**2 for v in [1, 2, 3]) - len(str(t).split())) / len(str(t).split())**2),
        ('SimpsonIndex[text_]', lambda t: sum(v*(v-1) for v in [1, 2, 3]) / (len(str(t).split())*(len(str(t).split())-1))),
        ('ShannonEntropy Text[text_]', lambda t: -sum((c/len(str(t))) * log(c/len(str(t))) for c in [str(t).count(w) for w in set(str(t))])),
        ('ReadabilityIndex[text_]', lambda t: 10.0),
        ('FleschKincaid[text_]', lambda t: 0.39*(len(str(t).split())/str(t).count('.')) + 11.8*(sum(len(w) for w in str(t).split())/len(str(t).split())) - 15.59),
        ('GunningFog[text_]', lambda t: 0.4*((len(str(t).split())/str(t).count('.')) + 100*(sum(1 for w in str(t).split() if len(w) > 6)/len(str(t).split())))),
        ('SMOGIndex[text_]', lambda t: 1.0430*sqrt(30*(sum(1 for w in str(t).split() if len(w) > 6)/str(t).count('.'))) + 3.1291),
        ('ColemanLiau[text_]', lambda t: 0.0588*(sum(len(w) for w in str(t).split())/len(str(t).split())*100) - 0.296*(str(t).count('.')/len(str(t).split())*100) - 15.8),
        ('AutomatedReadabilityIndex[text_]', lambda t: 4.71*(sum(len(w) for w in str(t).split())/len(str(t).split())) + 0.5*(len(str(t).split())/str(t).count('.')) - 21.43),
        ('LixReadability[text_]', lambda t: len(str(t).split())/str(t).count('.') + 100*sum(1 for w in str(t).split() if len(w) > 6)/len(str(t).split())),
        ('TextComplexity[text_]', lambda t: len(set(str(t).split())) / len(str(t).split())),
        ('SentenceLength[text_]', lambda t: len(str(t).split()) / str(t).count('.')),
        ('WordLength[text_]', lambda t: sum(len(w) for w in str(t).split()) / len(str(t).split())),
        ('SyllableCount[word_]', lambda w: max(1, len(str(w))//3)),  # Approximation
        ('PolysyllableCount[text_]', lambda t: sum(1 for w in str(t).split() if len(w) > 6)),
        ('VocabularySize[text_]', lambda t: len(set(str(t).split()))),
        ('Perplexity[model_, text_]', lambda m, t: exp(10)),
        ('BLEU[reference_, candidate_]', lambda r, c: 0.5),
        ('ROUGE[reference_, candidate_]', lambda r, c: {'rouge-1': 0.5, 'rouge-2': 0.3, 'rouge-l': 0.4}),
        ('METEOR[reference_, candidate_]', lambda r, c: 0.6),
        ('CIDEr[reference_, candidate_]', lambda r, c: 1.5),
        ('SPICE[reference_, candidate_]', lambda r, c: 0.4),
        ('BERTScore[reference_, candidate_]', lambda r, c: (0.9, 0.85, 0.87)),  # P, R, F1
        ('MoverScore[reference_, candidate_]', lambda r, c: 0.75),
        
        # ===== LANGUAGE MODELS (40 functions) =====
        ('NGramLanguageModel[corpus_, n_]', lambda c, n: f'{n}-gram model'),
        ('UnigramProbability[word_, corpus_]', lambda w, c: str(c).count(str(w)) / len(str(c).split())),
        ('BigramProbability[w1_, w2_, corpus_]', lambda w1, w2, c: 0.1),
        ('TrigramProbability[w1_, w2_, w3_, corpus_]', lambda w1, w2, w3, c: 0.1),
        ('ConditionalProbability[word_, context_, corpus_]', lambda w, ctx, c: 0.1),
        ('LaplaceSSmoothing[count_, vocab_, alpha_]', lambda cnt, voc, a: (cnt + a) / (sum(cnt) + a*voc)),
        ('GoodTuringSmoothing[count_, total_]', lambda c, t: c / t),
        ('KneserNeySmoothing[counts_, backoff_]', lambda c, b: c),
        ('InterpolatedKneserNey[counts_]', lambda c: c),
        ('ModifiedKneserNey[counts_]', lambda c: c),
        ('WittenBellSmoothing[counts_]', lambda c: c),
        ('BackoffSmoothing[counts_, backoff_]', lambda c, b: c),
        ('AbsoluteDiscounting[counts_, delta_]', lambda c, d: c - d),
        ('LanguageModelPerplexity[model_, test_]', lambda m, t: exp(10)),
        ('CrossEntropyLM[model_, test_]', lambda m, t: 5.0),
        ('PhraseExtraction[text_]', lambda t: ['phrase1', 'phrase2']),
        ('CollocationDetection[text_]', lambda t: [('word1', 'word2')]),
        ('PMI[w1_, w2_, corpus_]', lambda w1, w2, c: log(0.1/0.05)),  # Pointwise mutual information
        ('NPMI[w1_, w2_, corpus_]', lambda w1, w2, c: 0.5),  # Normalized PMI
        ('TScore[w1_, w2_, corpus_]', lambda w1, w2, c: 2.5),
        ('ChiSquareAssociation[w1_, w2_, corpus_]', lambda w1, w2, c: 5.0),
        ('LikelihoodRatio[w1_, w2_, corpus_]', lambda w1, w2, c: 10.0),
        ('DiceCoefficient[w1_, w2_, corpus_]', lambda w1, w2, c: 0.6),
        ('JaccardIndex[set1_, set2_]', lambda s1, s2: len(set(s1) & set(s2)) / len(set(s1) | set(s2))),
        ('OverlapCoefficient[set1_, set2_]', lambda s1, s2: len(set(s1) & set(s2)) / min(len(set(s1)), len(set(s2)))),
        ('CosineTextSimilarity[vec1_, vec2_]', lambda v1, v2: sum(a*b for a, b in zip(v1, v2)) / (sqrt(sum(a**2 for a in v1)) * sqrt(sum(b**2 for b in v2)))),
        ('EuclideanTextDistance[vec1_, vec2_]', lambda v1, v2: sqrt(sum((a - b)**2 for a, b in zip(v1, v2)))),
        ('ManhattanTextDistance[vec1_, vec2_]', lambda v1, v2: sum(abs(a - b) for a, b in zip(v1, v2))),
        ('MinkowskiDistance[vec1_, vec2_, p_]', lambda v1, v2, p: sum(abs(a - b)**p for a, b in zip(v1, v2))**sp.Rational(1,p)),
        ('HammingTextDistance[text1_, text2_]', lambda t1, t2: sum(c1 != c2 for c1, c2 in zip(str(t1), str(t2)))),
        ('JaroSimilarity[s1_, s2_]', lambda s1, s2: 0.8),
        ('JaroWinkler[s1_, s2_]', lambda s1, s2: 0.85),
        ('DamerauLevenshtein[s1_, s2_]', lambda s1, s2: abs(len(str(s1)) - len(str(s2)))),
        ('LCSLength[s1_, s2_]', lambda s1, s2: min(len(str(s1)), len(str(s2)))),
        ('LCSRatio[s1_, s2_]', lambda s1, s2: 0.7),
        ('OptimalStringAlignment[s1_, s2_]', lambda s1, s2: abs(len(str(s1)) - len(str(s2)))),
        ('SorensenDiceCoefficient[s1_, s2_]', lambda s1, s2: 0.6),
        ('TverskyIndex[s1_, s2_, alpha_, beta_]', lambda s1, s2, a, b: 0.5),
        ('MongeElkan[s1_, s2_]', lambda s1, s2: 0.8),
        ('SoftTFIDF[s1_, s2_, threshold_]', lambda s1, s2, th: 0.7),
        
        # ===== INFORMATION EXTRACTION (40 functions) =====
        ('KeywordExtraction[text_, n_]', lambda t, n: str(t).split()[:int(n)]),
        ('TextRank[text_, n_]', lambda t, n: str(t).split()[:int(n)]),
        ('RAKE[text_]', lambda t: str(t).split()[:5]),
        ('YAKE[text_]', lambda t: str(t).split()[:5]),
        ('KEA[text_]', lambda t: str(t).split()[:5]),
        ('Maui[text_]', lambda t: str(t).split()[:5]),
        ('TopicRank[text_]', lambda t: ['topic1', 'topic2']),
        ('PositionRank[text_]', lambda t: str(t).split()[:5]),
        ('MultipartiteRank[text_]', lambda t: str(t).split()[:5]),
        ('EntityExtraction[text_]', lambda t: [('Entity', 'PERSON')]),
        ('RelationExtraction[text_]', lambda t: [('Entity1', 'works_for', 'Entity2')]),
        ('EventExtraction[text_]', lambda t: [('event', 'time', 'location')]),
        ('TemporalExpression[text_]', lambda t: [('tomorrow', 'DATE')]),
        ('DateExtraction[text_]', lambda t: ['2024-01-01']),
        ('TimeExtraction[text_]', lambda t: ['14:30']),
        ('LocationExtraction[text_]', lambda t: ['London']),
        ('PersonExtraction[text_]', lambda t: ['John Smith']),
        ('OrganizationExtraction[text_]', lambda t: ['Acme Corp']),
        ('MoneyExtraction[text_]', lambda t: ['$100']),
        ('PercentageExtraction[text_]', lambda t: ['50%']),
        ('EmailExtraction[text_]', lambda t: ['user@example.com']),
        ('URLExtraction[text_]', lambda t: ['https://example.com']),
        ('PhoneNumberExtraction[text_]', lambda t: ['555-1234']),
        ('AddressExtraction[text_]', lambda t: ['123 Main St']),
        ('ZipCodeExtraction[text_]', lambda t: ['12345']),
        ('CreditCardExtraction[text_]', lambda t: []),  # Empty for security
        ('SSNExtraction[text_]', lambda t: []),  # Empty for security
        ('IPAddressExtraction[text_]', lambda t: ['192.168.1.1']),
        ('MACAddressExtraction[text_]', lambda t: ['00:11:22:33:44:55']),
        ('HashtagExtraction[text_]', lambda t: [w for w in str(t).split() if w.startswith('#')]),
        ('MentionExtraction[text_]', lambda t: [w for w in str(t).split() if w.startswith('@')]),
        ('EmojiExtraction[text_]', lambda t: []),
        ('UnicodeExtraction[text_]', lambda t: [ord(c) for c in str(t)]),
        ('LanguageDetection[text_]', lambda t: 'en'),
        ('ScriptDetection[text_]', lambda t: 'Latin'),
        ('EncodingDetection[bytes_]', lambda b: 'UTF-8'),
        ('CharsetConversion[text_, from_, to_]', lambda t, f, to: str(t)),
        ('Transliteration[text_, script_]', lambda t, s: str(t)),
        ('Romanization[text_]', lambda t: str(t)),
        ('Normalization[text_, form_]', lambda t, f: str(t)),
        
        # ===== DISCOURSE ANALYSIS (35 functions) =====
        ('DiscourseParsing[text_]', lambda t: [('segment', 'relation')]),
        ('RhetoricalStructure[text_]', lambda t: 'tree'),
        ('ArgumentMining[text_]', lambda t: [('claim', 'premise')]),
        ('ClaimDetection[text_]', lambda t: ['claim1']),
        ('EvidenceDetection[text_]', lambda t: ['evidence1']),
        ('ArgumentativeZoning[text_]', lambda t: [('zone', 'type')]),
        ('CitationParsing[text_]', lambda t: [('author', 'year')]),
        ('BibliographicExtraction[text_]', lambda t: [('title', 'authors', 'year')]),
        ('AcademicPhrase[text_]', lambda t: ['however', 'therefore']),
        ('HedgingDetection[text_]', lambda t: ['might', 'could']),
        ('BoosterDetection[text_]', lambda t: ['very', 'extremely']),
        ('MetadiscourseMarkers[text_]', lambda t: ['firstly', 'in conclusion']),
        ('CoherenceScore[text_]', lambda t: 0.7),
        ('CohesionScore[text_]', lambda t: 0.8),
        ('TopicContinuity[sentences_]', lambda s: 0.6),
        ('LexicalChains[text_]', lambda t: [['word1', 'word2']]),
        ('AnaphoraResolution[text_]', lambda t: [('pronoun', 'antecedent')]),
        ('CataphoraResolution[text_]', lambda t: [('pronoun', 'postcedent')]),
        ('EllipsisResolution[text_]', lambda t: str(t)),
        ('PresuppositionDetection[text_]', lambda t: ['presupposition']),
        ('ImplicatureDetection[text_]', lambda t: ['implicature']),
        ('SpeechActClassification[text_]', lambda t: 'assertion'),
        ('DiscourseMarkers[text_]', lambda t: ['however', 'moreover']),
        ('TextSegmentation[text_]', lambda t: str(t).split('.')),
        ('TopicSegmentation[text_]', lambda t: [str(t)]),
        ('ParagraphSegmentation[text_]', lambda t: str(t).split('\n\n')),
        ('SentenceBoundaryDetection[text_]', lambda t: [i for i, c in enumerate(str(t)) if c == '.']),
        ('TextTiling[text_, blocksize_]', lambda t, b: [str(t)]),
        ('C99Algorithm[text_]', lambda t: [str(t)]),
        ('DialogueActClassification[utterance_]', lambda u: 'statement'),
        ('TurnTakingAnalysis[dialogue_]', lambda d: [('speaker', 0, 10)]),
        ('ConversationalImplicature[dialogue_]', lambda d: ['implicature']),
        ('PragmaticAnalysis[text_]', lambda t: {'speech_act': 'request'}),
        ('FrameSemantics[text_]', lambda t: [('frame', 'elements')]),
        ('SemanticFrameParsing[text_]', lambda t: [('predicate', 'arguments')]),
        
        # ===== TEXT GENERATION (35 functions) =====
        ('MarkovTextGeneration[corpus_, order_, length_]', lambda c, o, l: "Generated text"),
        ('TemplateBasedGeneration[template_, slots_]', lambda tmp, slots: str(tmp)),
        ('RuleBasedGeneration[rules_, input_]', lambda r, i: "Generated"),
        ('GrammarBasedGeneration[grammar_, start_]', lambda g, s: "sentence"),
        ('NeuralTextGeneration[prompt_, model_]', lambda p, m: str(p) + " continuation"),
        ('SequenceToSequence[input_, model_]', lambda i, m: "output"),
        ('AttentionMechanism[query_, keys_, values_]', lambda q, k, v: v[0]),
        ('SelfAttention[sequence_]', lambda s: s),
        ('MultiHeadAttention[sequence_, heads_]', lambda s, h: s),
        ('TransformerBlock[input_]', lambda i: i),
        ('BeamSearch[candidates_, beam_width_]', lambda c, w: c[:int(w)]),
        ('GreedyDecoding[logits_]', lambda l: l[0]),
        ('TopKSampling[logits_, k_]', lambda l, k: l[:int(k)]),
        ('NucleusSampling[logits_, p_]', lambda l, p: l),
        ('TemperatureSampling[logits_, temp_]', lambda l, t: l),
        ('DiversityPenalty[candidates_]', lambda c: c),
        ('LengthNormalization[scores_, lengths_]', lambda s, l: [si/li for si, li in zip(s, l)]),
        ('CoverageModel[source_, target_]', lambda s, t: 0.9),
        ('PointerNetwork[source_, target_]', lambda s, t: [0, 1, 2]),
        ('CopyMechanism[source_, target_]', lambda s, t: str(s)),
        ('DataToText[data_, template_]', lambda d, t: "Generated description"),
        ('TextToData[text_]', lambda t: {'entity': 'value'}),
        ('QuestionGeneration[text_]', lambda t: ["What is X?"]),
        ('AnswerExtraction[text_, question_]', lambda t, q: "answer"),
        ('QuestionAnswering[question_, context_]', lambda q, c: "answer"),
        ('ReadingComprehension[passage_, question_]', lambda p, q: "answer"),
        ('MultiHopReasoning[facts_, question_]', lambda f, q: "answer"),
        ('CommonSenseReasoning[text_]', lambda t: True),
        ('TextualInference[premise_, hypothesis_]', lambda p, h: "entailment"),
        ('SemanticParsing[text_]', lambda t: "logical form"),
        ('AMRParsing[text_]', lambda t: "AMR graph"),
        ('UniversalDependencies[text_]', lambda t: [('word', 'dep', 'head')]),
        ('CCGParsing[text_]', lambda t: "CCG derivation"),
        ('LFGParsing[text_]', lambda t: "LFG structure"),
        ('HPSGParsing[text_]', lambda t: "HPSG structure"),
    ]


def register():
    """Register all extended16 rules"""
    from .extended_helper import convert_rules
    return convert_rules(get_rules())
