"""
mdmin — Rule-based Markdown Compression Engine
Targets 20-35% token reduction on typical LLM input.
"""
from __future__ import annotations

import re
from dataclasses import dataclass
from typing import Literal

from ._synonyms import apply_token_synonyms

_I = re.IGNORECASE
_M = re.MULTILINE
_IM = re.IGNORECASE | re.MULTILINE

# ---------------------------------------------------------------------------
# Verbose Pattern Database
# ---------------------------------------------------------------------------
# Format: (compiled_pattern, replacement_str_or_callable)

_VERBOSE_LIGHT: list[tuple] = [
    (re.compile(r"\bIn order to\b"),                         "To"),
    (re.compile(r"\bIt is important to note that\b", _I),    "Note:"),
    (re.compile(r"\bPlease note that\b", _I),                "Note:"),
    (re.compile(r"\bAs a matter of fact,?\s*", _I),          "In fact, "),
    (re.compile(r"\bAt this point in time\b", _I),           "Now"),
    (re.compile(r"\bDue to the fact that\b", _I),            "Because"),
    (re.compile(r"\bIn the event that\b", _I),               "If"),
    (re.compile(r"\bFor the purpose of\b", _I),              "To"),
    (re.compile(r"\bWith regard to\b", _I),                  "Re:"),
    (re.compile(r"\bWith respect to\b", _I),                 "Re:"),
    (re.compile(r"\bIn spite of the fact that\b", _I),       "Despite"),
    (re.compile(r"\bIt should be noted that\b", _I),         ""),
    (re.compile(r"\bThere is no doubt that\b", _I),          ""),
    (re.compile(r"\bAs previously mentioned,?\s*", _I),      ""),
    (re.compile(r"\bAt the present time\b", _I),             "Currently"),
    (re.compile(r"\bPrior to\b"),                            "Before"),
    (re.compile(r"\bSubsequent to\b"),                       "After"),
    (re.compile(r"\bIn the near future\b", _I),              "Soon"),
    (re.compile(r"\bIn lieu of\b", _I),                      "Instead of"),
    (re.compile(r"\bIn the absence of\b", _I),               "Without"),
    (re.compile(r"\bOn the basis of\b", _I),                 "Based on"),
    (re.compile(r"\bIn the case of\b", _I),                  "For"),
    (re.compile(r"\bBy means of\b", _I),                     "By"),
    (re.compile(r"\bIn terms of\b", _I),                     "In"),
    (re.compile(r"\bWith the exception of\b", _I),           "Except"),
    (re.compile(r"\bIn the context of\b", _I),               "In"),
    (re.compile(r"\bFor the sake of\b", _I),                 "For"),
    (re.compile(r"\bIn accordance with\b", _I),              "Per"),
    (re.compile(r"\bIn conjunction with\b", _I),             "With"),
    (re.compile(r"\bAs a consequence of\b", _I),             "Due to"),
    (re.compile(r"\bIn the majority of cases\b", _I),        "Usually"),
    (re.compile(r"\bOn the grounds that\b", _I),             "Because"),
    (re.compile(r"\bIn order for\b", _I),                    "For"),
    (re.compile(r"\bWith the goal of\b", _I),                "To"),
    (re.compile(r"\bWith the aim of\b", _I),                 "To"),
    (re.compile(r"\bWith the objective of\b", _I),           "To"),
    (re.compile(r"\bWith a view to\b", _I),                  "To"),
    (re.compile(r"\bFor the reason that\b", _I),             "Because"),
    (re.compile(r"\bOn the condition that\b", _I),           "If"),
    (re.compile(r"\bIn the form of\b", _I),                  "as"),
    (re.compile(r"\bBy way of\b", _I),                       "via"),
    (re.compile(r"\bIn the situation where\b", _I),          "When"),
    (re.compile(r"\bIn the event of\b", _I),                 "If"),
    (re.compile(r"\bAt the time of\b", _I),                  "When"),
    (re.compile(r"\bWith reference to\b", _I),               "Re:"),
]

_VERBOSE_MEDIUM: list[tuple] = [
    (re.compile(r"\bIn addition to this,?\s*", _I),          "Also, "),
    (re.compile(r"\bIn addition,?\s*", _I),                  "Also, "),
    (re.compile(r"\bFurthermore,?\s*", _I),                  "Also, "),
    (re.compile(r"\bMoreover,?\s*", _I),                     "Also, "),
    (re.compile(r"\bAdditionally,?\s*", _I),                 "Also, "),
    (re.compile(r"\bOn the other hand,?\s*", _I),            "But, "),
    (re.compile(r"\bNevertheless,?\s*", _I),                 "But, "),
    (re.compile(r"\bNonetheless,?\s*", _I),                  "But, "),
    (re.compile(r"\bAs a result of\b", _I),                  "Due to"),
    (re.compile(r"\bAs a result,?\s*", _I),                  "So, "),
    (re.compile(r"\bConsequently,?\s*", _I),                 "So, "),
    (re.compile(r"\bA large number of\b", _I),               "Many"),
    (re.compile(r"\bA significant number of\b", _I),         "Many"),
    (re.compile(r"\bA considerable amount of\b", _I),        "Much"),
    (re.compile(r"\b(?:Has|Have) the ability to\b", _I),     "Can"),
    (re.compile(r"\b(?:Is|Are) able to\b", _I),              "Can"),
    (re.compile(r"\b(?:Was|Were) able to\b", _I),            "Could"),
    (re.compile(r"\b(?:Has|Have) the capacity to\b", _I),    "Can"),
    (re.compile(r"\b(?:Has|Have) the capability to\b", _I),  "Can"),
    (re.compile(r"\b(?:Has|Have) the opportunity to\b", _I), "Can"),
    (re.compile(r"\b(?:Has|Have) the potential to\b", _I),   "Can"),
    (re.compile(r"\bIn the process of\b", _I),               "While"),
    (re.compile(r"\bTake into consideration\b", _I),         "Consider"),
    (re.compile(r"\bTake into account\b", _I),               "Consider"),
    (re.compile(r"\bMake a decision\b", _I),                 "Decide"),
    (re.compile(r"\bCome to the conclusion\b", _I),          "Conclude"),
    (re.compile(r"\bCome to a decision\b", _I),              "Decide"),
    (re.compile(r"\bGive an indication of\b", _I),           "Indicate"),
    (re.compile(r"\bHave a tendency to\b", _I),              "Tend to"),
    (re.compile(r"\bIn close proximity to\b", _I),           "Near"),
    (re.compile(r"\bA number of\b", _I),                     "Several"),
    (re.compile(r"\bThe vast majority of\b", _I),            "Most"),
    (re.compile(r"\bThe majority of\b", _I),                 "Most"),
    (re.compile(r"\bOn a daily basis\b", _I),                "Daily"),
    (re.compile(r"\bOn a regular basis\b", _I),              "Regularly"),
    (re.compile(r"\bOn a weekly basis\b", _I),               "Weekly"),
    (re.compile(r"\bOn a monthly basis\b", _I),              "Monthly"),
    (re.compile(r"\bAt the end of the day,?\s*", _I),        "Ultimately, "),
    (re.compile(r"\bEach and every\b", _I),                  "Every"),
    (re.compile(r"\bFirst and foremost,?\s*", _I),           "First, "),
    (re.compile(r"\bBasically,?\s*", _I),                    ""),
    (re.compile(r"\bEssentially,?\s*", _I),                  ""),
    (re.compile(r"\bFundamentally,?\s*", _I),                ""),
    (re.compile(r"\bObviously,?\s*", _I),                    ""),
    (re.compile(r"\bClearly,?\s*", _I),                      ""),
    (re.compile(r"\bOf course,?\s*", _I),                    ""),
    (re.compile(r"\bNeedless to say,?\s*", _I),              ""),
    (re.compile(r"\bIt goes without saying that\b", _I),     ""),
    (re.compile(r"\bAs you (?:may |might |probably )?know,?\s*", _I), ""),
    (re.compile(r"\bAs (?:we|you) (?:can |might |may )?see,?\s*", _I), ""),
    (re.compile(r"\bIt is worth (?:mentioning|noting) that\b", _I), ""),
    (re.compile(r"\bIt is important to (?:understand|remember|realize) that\b", _I), ""),
    (re.compile(r"\bPlease (?:keep|bear) in mind that\b", _I), ""),
    (re.compile(r"\bThere is no question that\b", _I),       ""),
    (re.compile(r"\bThe fact of the matter is(?: that)?\b", _I), ""),
    (re.compile(r"\bWhat this means is(?: that)?\b", _I),    ""),
    (re.compile(r"\bThe bottom line is(?: that)?\b", _I),    ""),
    (re.compile(r"\bThe point is(?: that)?\b", _I),          ""),
    (re.compile(r"\bThe thing is(?: that)?,?\s*", _I),       ""),
    (re.compile(r"\bHaving said that,?\s*", _I),             ""),
    (re.compile(r"\bThat being said,?\s*", _I),              ""),
    (re.compile(r"\bWith that being said,?\s*", _I),         ""),
    (re.compile(r"\bAll things considered,?\s*", _I),        "Overall, "),
    (re.compile(r"\bBy and large,?\s*", _I),                 "Generally, "),
    (re.compile(r"\bFor the most part,?\s*", _I),            "Mostly, "),
    (re.compile(r"\bTo a large extent,?\s*", _I),            "Largely, "),
    (re.compile(r"\bTo a certain extent,?\s*", _I),          "Partly, "),
    (re.compile(r"\bIt is worth pointing out that\b", _I),   ""),
    (re.compile(r"\bOne of the things that\b", _I),          "Something that"),
    (re.compile(r"\bThere are a number of\b", _I),           "Several"),
    (re.compile(r"\bThere are many\b", _I),                  "Many"),
    (re.compile(r"\bThere are several\b", _I),               "Several"),
    (re.compile(r"\bThere is a\b", _I),                      "A"),
    (re.compile(r"\bIt is (?:also )?possible to\b", _I),     "Can"),
    (re.compile(r"\bIt is (?:also )?necessary to\b", _I),    "Must"),
    (re.compile(r"\bIt is recommended (?:that |to )", _I),   "Recommend "),
    (re.compile(r"\bMake sure (?:that |to )", _I),           "Ensure "),
    (re.compile(r"\bIn this (?:section|document|article|guide|tutorial),?\s*(?:we will |we'll |we )?(?:discuss |describe |explain |cover |look at |go over |explore |examine )?\s*", _I), ""),
    (re.compile(r"\bLet's (?:take a look at|look at|examine|explore|dive into|discuss)\b", _I), ""),
    # Nominalization reversal
    (re.compile(r"\b(?:make|makes|made|making) use of\b", _I),             "use"),
    (re.compile(r"\b(?:make|makes|made|making) an? attempt to\b", _I),     "try to"),
    (re.compile(r"\b(?:make|makes|made|making) an? effort to\b", _I),      "try to"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:improvement|improvements) to\b", _I), "improve"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:change|changes) to\b", _I), "change"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:adjustment|adjustments) to\b", _I), "adjust"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:modification|modifications) to\b", _I), "modify"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:contribution|contributions) to\b", _I), "contribute to"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?(?:recommendation|recommendations)\b", _I), "recommend"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?assessment\b", _I), "assess"),
    (re.compile(r"\b(?:make|makes|made|making) (?:an? )?evaluation\b", _I), "evaluate"),
    (re.compile(r"\b(?:make|makes|made|making) reference to\b", _I),       "reference"),
    (re.compile(r"\b(?:make|makes|made|making) contact with\b", _I),       "contact"),
    (re.compile(r"\b(?:have|has|had|having) an? impact on\b", _I),         "impact"),
    (re.compile(r"\b(?:have|has|had|having) an? effect on\b", _I),         "affect"),
    (re.compile(r"\b(?:have|has|had|having) an? influence on\b", _I),      "influence"),
    (re.compile(r"\b(?:have|has|had) an? awareness of\b", _I),             "know"),
    (re.compile(r"\b(?:have|has|had) an? understanding of\b", _I),         "understand"),
    (re.compile(r"\b(?:have|has|had) knowledge of\b", _I),                 "know"),
    (re.compile(r"\b(?:have|has|had) a role in\b", _I),                    "role in"),
    (re.compile(r"\b(?:provide|provides|provided|providing) (?:support|help|assistance) (?:for|to)\b", _I), "support"),
    (re.compile(r"\b(?:provide|provides|provided|providing) guidance (?:to|for)?\b", _I), "guide"),
    (re.compile(r"\b(?:provide|provides|provided|providing) (?:an? )?(?:overview|summary) of\b", _I), "overview:"),
    (re.compile(r"\b(?:provide|provides|provided|providing) (?:an? )?description of\b", _I), "describe"),
    (re.compile(r"\b(?:provide|provides|provided|providing) (?:an? )?explanation of\b", _I), "explain"),
    (re.compile(r"\b(?:provide|provides|provided|providing) access to\b", _I), "allow access to"),
    (re.compile(r"\b(?:give|gives|gave|giving) consideration to\b", _I),   "consider"),
    (re.compile(r"\b(?:give|gives|gave|giving) priority to\b", _I),        "prioritize"),
    (re.compile(r"\b(?:give|gives|gave|giving) attention to\b", _I),       "focus on"),
    (re.compile(r"\b(?:give|gives|gave|giving) (?:an? )?(?:indication|idea) of\b", _I), "indicate"),
    (re.compile(r"\b(?:take|takes|took|taking) advantage of\b", _I),       "use"),
    (re.compile(r"\b(?:take|takes|took|taking) responsibility for\b", _I), "own"),
    (re.compile(r"\b(?:take|takes|took|taking) care of\b", _I),            "handle"),
    (re.compile(r"\b(?:take|takes|took|taking) (?:action|steps|measures) (?:to|on|against)?\b", _I), "act"),
    (re.compile(r"\b(?:take|takes|took|taking) (?:a |the )?look at\b", _I), "examine"),
    (re.compile(r"\b(?:take|takes|took|taking) part in\b", _I),            "join"),
    (re.compile(r"\b(?:take|takes|took|taking) place\b", _I),              "occur"),
    (re.compile(r"\b(?:carry|carries|carried|carrying) out (?:a |an |the )?(review|analysis|study|test|evaluation|investigation|assessment)\b", _I), lambda m: m.group(1)),
    (re.compile(r"\b(?:carry|carries|carried|carrying) out\b", _I),        "do"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?review\b", _I), "review"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?(?:analysis|analyses)\b", _I), "analyze"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?study\b", _I), "study"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?testing?\b", _I), "test"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?survey\b", _I), "survey"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?evaluation\b", _I), "evaluate"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) (?:a |an |the )?investigation\b", _I), "investigate"),
    (re.compile(r"\b(?:conduct|conducts|conducted|conducting) research\b", _I), "research"),
    (re.compile(r"\b(?:perform|performs|performed|performing) (?:a |an |the )?review\b", _I), "review"),
    (re.compile(r"\b(?:perform|performs|performed|performing) (?:a |an |the )?analysis\b", _I), "analyze"),
    (re.compile(r"\b(?:perform|performs|performed|performing) (?:a |an |the )?testing?\b", _I), "test"),
    (re.compile(r"\b(?:perform|performs|performed|performing) (?:a |an |the )?assessment\b", _I), "assess"),
    (re.compile(r"\b(?:perform|performs|performed|performing) (?:a |an |the )?evaluation\b", _I), "evaluate"),
    (re.compile(r"\bbe in a position to\b", _I),   "can"),
    (re.compile(r"\bbe in the process of\b", _I),  "be"),
    (re.compile(r"\bbe aware of\b", _I),           "know"),
    (re.compile(r"\bbe capable of\b", _I),         "can"),
    (re.compile(r"\bput in place\b", _I),          "implement"),
    (re.compile(r"\bput forward\b", _I),           "propose"),
    (re.compile(r"\bput emphasis on\b", _I),       "emphasize"),
    (re.compile(r"\bas well as\b", _I),            "and"),
    (re.compile(r"\bat (?:a |the )?later (?:date|time|point)\b", _I), "later"),
    (re.compile(r"\bon (?:a |the )?(?:ongoing|continuous|regular) basis\b", _I), "continuously"),
    (re.compile(r"\bon (?:a |the )?case[- ]by[- ]case basis\b", _I), "per case"),
    # Qualifier removal
    (re.compile(r"\bvery (important|critical|crucial|essential|significant|relevant|useful|helpful|effective|efficient|clear|simple|complex|large|small|fast|slow|easy|difficult|hard|common|rare)\b", _I), r"\1"),
    (re.compile(r"\bhighly (important|critical|crucial|essential|significant|relevant|useful|helpful|effective|efficient|recommended|available|scalable|reliable|secure|optimized|configurable|customizable)\b", _I), r"\1"),
    (re.compile(r"\bextremely (important|critical|crucial|essential|significant|useful|effective|efficient|complex|simple|large|small|fast|slow|difficult|hard)\b", _I), r"\1"),
    (re.compile(r"\bincredibly (important|useful|effective|efficient|powerful|flexible|scalable|reliable)\b", _I), r"\1"),
    (re.compile(r"\bparticularly (important|useful|effective|relevant|critical|notable)\b", _I), r"\1"),
    (re.compile(r"\brelatively (simple|easy|straightforward|complex|fast|slow|new|old|small|large)\b", _I), r"\1"),
    (re.compile(r"\bconsiderable (amount|number|portion|degree|impact|improvement)\b", _I), r"\1"),
    (re.compile(r"\bsignificant (amount|number|portion|degree|impact|improvement|reduction|increase|decrease)\b", _I), r"\1"),
]

_VERBOSE_AGGRESSIVE: list[tuple] = [
    (re.compile(r"\bThe following is a list of\b", _I), ""),
    (re.compile(r"\bThe following (?:are|is)\b", _I),   "These are"),
    (re.compile(r"\bThe following\b", _I),              "These"),
    (re.compile(r"\bAs mentioned (?:earlier|above|previously|before),?\s*", _I), ""),
    (re.compile(r"\bAs (?:described|discussed|noted|stated|outlined) (?:earlier|above|previously|before),?\s*", _I), ""),
    (re.compile(r"\bThis is because\b", _I),            "Because"),
    (re.compile(r"\bThe reason (?:for this )?(?:is|being)(?: that)?\b", _I), "Because"),
    (re.compile(r"\bTo put it simply,?\s*", _I),        ""),
    (re.compile(r"\bTo put it another way,?\s*", _I),   "i.e., "),
    (re.compile(r"\bIn other words,?\s*", _I),          "i.e., "),
    (re.compile(r"\bThat is to say,?\s*", _I),          "i.e., "),
    (re.compile(r"\bSo to speak,?\s*", _I),             ""),
    (re.compile(r"\bAs such,?\s*", _I),                 "So, "),
    (re.compile(r"\bGiven that\b", _I),                 "Since"),
    (re.compile(r"\bOwing to\b", _I),                   "Due to"),
    (re.compile(r"\bProvided that\b", _I),              "If"),
    (re.compile(r"\bAssuming that\b", _I),              "If"),
    (re.compile(r"\bIn the same way\b", _I),            "Similarly"),
    (re.compile(r"\bBy the same token,?\s*", _I),       "Similarly, "),
    (re.compile(r"\bLast but not least,?\s*", _I),      "Finally, "),
    (re.compile(r"\bTo summarize,?\s*", _I),            ""),
    (re.compile(r"\bTo sum up,?\s*", _I),               ""),
    (re.compile(r"\bIn summary,?\s*", _I),              ""),
    (re.compile(r"\bIn conclusion,?\s*", _I),           ""),
    (re.compile(r"\bTo conclude,?\s*", _I),             ""),
    # Hedge/filler words
    (re.compile(r"\bbasically\b", _I),   ""),
    (re.compile(r"\bactually\b", _I),    ""),
    (re.compile(r"\bjust\b", _I),        ""),
    (re.compile(r"\breally\b", _I),      ""),
    (re.compile(r"\bsimply\b", _I),      ""),
    (re.compile(r"\bquite\b", _I),       ""),
    (re.compile(r"\brather\b", _I),      ""),
    (re.compile(r"\bsomewhat\b", _I),    ""),
    (re.compile(r"\bperhaps\b", _I),     "maybe"),
    (re.compile(r"\bvirtually\b", _I),   ""),
    (re.compile(r"\brespectively\b", _I), ""),
    (re.compile(r"\boverall\b", _I),     ""),
    (re.compile(r"\btypically\b", _I),   ""),
    (re.compile(r"\bnormally\b", _I),    ""),
    (re.compile(r"\bvery\b", _I),        ""),
    (re.compile(r"\bextremely\b", _I),   ""),
    (re.compile(r"\bhighly\b", _I),      ""),
    (re.compile(r"\bincredibly\b", _I),  ""),
    (re.compile(r"\bnoticeably\b", _I),  ""),
    (re.compile(r"\bsignificantly\b", _I), ""),
    (re.compile(r"\bconsiderably\b", _I), ""),
    (re.compile(r"\bsubstantially\b", _I), ""),
    (re.compile(r"\bgreatly\b", _I),     ""),
    (re.compile(r"\bexcessively\b", _I), ""),
    (re.compile(r"\boverly\b", _I),      ""),
    (re.compile(r"\bunduly\b", _I),      ""),
    # Common compressions
    (re.compile(r"\band so on\b", _I),   "etc."),
    (re.compile(r"\band so forth\b", _I), "etc."),
    (re.compile(r"\bet cetera\b", _I),   "etc."),
    (re.compile(r"\bfor example\b", _I), "e.g."),
    (re.compile(r"\bfor instance\b", _I), "e.g."),
    (re.compile(r"\bthat is\b", _I),     "i.e."),
    (re.compile(r"\bsuch as\b", _I),     "e.g."),
    (re.compile(r"\bincluding but not limited to\b", _I), "including"),
    (re.compile(r"\bup to and including\b", _I), "through"),
    (re.compile(r"\bin the range of\b", _I), "~"),
    (re.compile(r"\bapproximately\b", _I), "~"),
    # Filler sentence starters
    (re.compile(r"\bIt is clear that\b", _I),    ""),
    (re.compile(r"\bIt is evident that\b", _I),  ""),
    (re.compile(r"\bIt is apparent that\b", _I), ""),
    (re.compile(r"\bIt is obvious that\b", _I),  ""),
    (re.compile(r"\bWe can see that\b", _I),     ""),
    (re.compile(r"\bWe believe that\b", _I),     ""),
    (re.compile(r"\bWe have found that\b", _I),  ""),
    (re.compile(r"\bIt has been determined that\b", _I), ""),
    (re.compile(r"\bIt has been found that\b", _I), ""),
    (re.compile(r"\bIt can be seen that\b", _I), ""),
    (re.compile(r"\bIt can be observed that\b", _I), ""),
    (re.compile(r"\bThis means that\b", _I),     ""),
    (re.compile(r"\bThis implies that\b", _I),   ""),
    (re.compile(r"\bThis indicates that\b", _I), ""),
    (re.compile(r"\bThis suggests that\b", _I),  ""),
    (re.compile(r"\bAs a general rule,?\s*", _I), ""),
    (re.compile(r"\bIn general,?\s*", _I),       ""),
    (re.compile(r"\bGenerally speaking,?\s*", _I), ""),
    (re.compile(r"\bBroadly speaking,?\s*", _I), ""),
    (re.compile(r"\bthe following\b", _I),       "these"),
    (re.compile(r"\bthe same\b", _I),            "same"),
    # is/are compressions
    (re.compile(r"\b(?:is|are) capable of\b", _I), "can"),
    (re.compile(r"\b(?:is|are|was|were) (?:also |)?(?:known|referred to) as\b", _I), "called"),
    (re.compile(r"\b(is|are|was|were) considered (?:to be |as )?\b", _I), r"\1 "),
    (re.compile(r"\b(is|are|was|were) (?:seen|regarded|viewed) as\b", _I), r"\1"),
    (re.compile(r"\b(?:is|are) defined as\b", _I), "means"),
    (re.compile(r"\b(?:is|are|was|were) (?:required|expected|supposed|meant|intended) to\b", _I), "must"),
    (re.compile(r"\b(?:is|are|was|were) believed to be\b", _I), "may be"),
    (re.compile(r"\b(?:is|are|was|were) thought to be\b", _I), "may be"),
    (re.compile(r"\b(?:is|are|was|were) assumed to be\b", _I), "may be"),
    (re.compile(r"\b(?:is|are) used (?:for|to)\b", _I), "handles"),
    # Back-references
    (re.compile(r",?\s*\(as (?:mentioned|noted|discussed|described|outlined|stated) (?:above|earlier|previously|before|below)\)", _I), ""),
    (re.compile(r",?\s*as (?:mentioned|noted|discussed|described|outlined|stated) (?:above|earlier|previously|before)\b", _I), ""),
    (re.compile(r"\(see (?:above|below|section|chapter|figure|table)[^)]{0,40}\)", _I), ""),
    (re.compile(r"\(refer to (?:section|chapter|figure|table)[^)]{0,40}\)", _I), ""),
    (re.compile(r"\(for more (?:details|information|context)[^)]{0,40}\)", _I), ""),
    (re.compile(r"\(for (?:details|information|context)[^)]{0,40}\)", _I), ""),
    # Redundant doc structure
    (re.compile(r"\b(?:This|The) (?:section|chapter|document|guide|article|tutorial|page) (?:also )?(?:discusses?|describes?|explains?|covers?|provides?|presents?|outlines?|reviews?|introduces?|walks? through)\b", _I), ""),
    (re.compile(r"\bThe (?:rest|remainder) of this (?:document|section|guide|chapter)\b", _I), "Below"),
    (re.compile(r"\bIn the following (?:section|sections|chapter|chapters|pages?)\b", _I), "Below"),
    (re.compile(r"\bBuilding on (?:this|that),?\s*", _I), ""),
    (re.compile(r"\bWith this (?:in mind|knowledge|background|understanding),?\s*", _I), ""),
    (re.compile(r"\bBefore (?:we|you) (?:dive into|continue|proceed|move on|start|begin),?\s*", _I), ""),
    (re.compile(r"\bThroughout this (?:document|guide|section|chapter),?\s*", _I), ""),
    (re.compile(r"\bAt (?:a |the )?high level,?\s*", _I), "Broadly, "),
    (re.compile(r"\bAs (?:we|you) (?:will |can )?(?:see|note|observe|learn|discover|find) (?:below|above|here|in this (?:section|document)),?\s*", _I), ""),
    # More
    (re.compile(r"\bnot only (.+?) but also\b", _I), r"\1 and"),
    (re.compile(r"\b(believe|know|think|note|ensure|said|found|assume|suggest|show|indicate|confirm|expect) that\b", _I), r"\1"),
    (re.compile(r"\b(?:is|are) not (?:a )?feasible\b", _I), "can't work"),
    (re.compile(r"\bhas been proven to be\b", _I),  "is"),
    (re.compile(r"\bhave been proven to be\b", _I), "are"),
    (re.compile(r"\bwill continue to\b", _I),       "will"),
    (re.compile(r"\bgoing forward\b", _I),          ""),
    (re.compile(r"\bat any given (?:point in )?time\b", _I), "anytime"),
    (re.compile(r"\bin this case,?\s*", _I),        ""),
    (re.compile(r"\bin this scenario,?\s*", _I),    ""),
    (re.compile(r"\bin this situation,?\s*", _I),   ""),
    (re.compile(r"\bat this stage,?\s*", _I),       ""),
    (re.compile(r"\bit is worth noting that\b", _I), ""),
    (re.compile(r"\bit is important to note that\b", _I), "Note:"),
    (re.compile(r"\bthe reason (?:for this|why) (?:is|being)\b", _I), "Because"),
]

_VERBOSE_ULTRA: list[tuple] = [
    (re.compile(r"\bthe ([A-Z]{2,})\b"),  r"\1"),
    (re.compile(r"\bthe ([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b"), r"\1"),
    (re.compile(r"\bour system\b", _I),       "system"),
    (re.compile(r"\bour services?\b", _I),    "services"),
    (re.compile(r"\bour infrastructure\b", _I), "infrastructure"),
    (re.compile(r"\bour architecture\b", _I), "architecture"),
    (re.compile(r"\bour deployment\b", _I),   "deployment"),
    (re.compile(r"\bour pipeline\b", _I),     "pipeline"),
    (re.compile(r"\bour team\b", _I),         "team"),
    (re.compile(r"\bour application\b", _I),  "application"),
    (re.compile(r"\bour platform\b", _I),     "platform"),
    (re.compile(r"\bour codebase\b", _I),     "codebase"),
    (re.compile(r"\bour database\b", _I),     "database"),
    (re.compile(r"\bour API\b"),              "API"),
    (re.compile(r"\bwe use\b", _I),           "uses"),
    (re.compile(r"\bwe utilize\b", _I),       "uses"),
    (re.compile(r"\bwe implement(?:ed)?\b", _I), "implements"),
    (re.compile(r"\bwe maintain\b", _I),      "maintains"),
    (re.compile(r"\bwe run\b", _I),           "runs"),
    (re.compile(r"\bwe deploy\b", _I),        "deploys"),
    (re.compile(r"\bwe need to\b", _I),       "must"),
    (re.compile(r"\bwe want to\b", _I),       "should"),
    (re.compile(r"\bwe aim to\b", _I),        "aims to"),
    (re.compile(r"\bwe strive to\b", _I),     "strives to"),
    (re.compile(r"\bwe recommend\b", _I),     "recommend"),
    (re.compile(r"\bwe support\b", _I),       "supports"),
    (re.compile(r"\bwe provide\b", _I),       "provides"),
    (re.compile(r"\bwe handle\b", _I),        "handles"),
    (re.compile(r"\bwe store\b", _I),         "stores"),
    (re.compile(r"\bwe manage\b", _I),        "manages"),
    (re.compile(r"\bwe expose\b", _I),        "exposes"),
    (re.compile(r"\bwe validate\b", _I),      "validates"),
    (re.compile(r"\bwe track\b", _I),         "tracks"),
    (re.compile(r"\bwe monitor\b", _I),       "monitors"),
    (re.compile(r"\bwe log\b", _I),           "logs"),
    (re.compile(r"\bwe cache\b", _I),         "caches"),
    (re.compile(r"\bwe process\b", _I),       "processes"),
    (re.compile(r"\bwe send\b", _I),          "sends"),
    (re.compile(r"\bwe receive\b", _I),       "receives"),
    (re.compile(r"\bwe return\b", _I),        "returns"),
    (re.compile(r"\bwe generate\b", _I),      "generates"),
    (re.compile(r"\bwe create\b", _I),        "creates"),
    (re.compile(r"\bwe delete\b", _I),        "deletes"),
    (re.compile(r"\bwe update\b", _I),        "updates"),
    (re.compile(r"\bwe fetch\b", _I),         "fetches"),
    (re.compile(r"\bwe encrypt\b", _I),       "encrypts"),
    (re.compile(r"\bwe authenticate\b", _I),  "authenticates"),
    (re.compile(r"\bwe authorize\b", _I),     "authorizes"),
    (re.compile(r"\bscalability and maintainability\b", _I), "scalability+maintainability"),
    (re.compile(r"\bauthentication and authorization\b", _I), "auth/authz"),
    (re.compile(r"\bunit tests,? integration tests,? and end-to-end tests\b", _I), "unit/integration/e2e tests"),
    (re.compile(r"\bread and write\b", _I),   "read/write"),
    (re.compile(r"\bencryption and decryption\b", _I), "encryption/decryption"),
    (re.compile(r"\brequest and response\b", _I), "req/res"),
    (re.compile(r"\binput and output\b", _I), "I/O"),
    (re.compile(r"\bfront[- ]end and back[- ]end\b", _I), "frontend/backend"),
    (re.compile(r"\b(believe|know|think|note|ensure|said|found|assume|suggest|show|indicate|confirm|expect) that\b", _I), r"\1"),
    (re.compile(r"\bthat have been\b", _I),   ""),
    (re.compile(r"\bthat has been\b", _I),    ""),
    (re.compile(r"\bthat are\b", _I),         ""),
    (re.compile(r"\bwhich are\b", _I),        ""),
    (re.compile(r"\bwhich is\b", _I),         ""),
    (re.compile(r"\bhave made a decision to\b", _I), "decided to"),
    (re.compile(r"\bhas made a decision to\b", _I),  "decided to"),
    (re.compile(r"\bneed(?:s)? to be taken into consideration\b", _I), "to consider"),
    (re.compile(r"\btaken into consideration\b", _I), "considered"),
    (re.compile(r"\bnot only (.+?) but also\b", _I), r"\1 and"),
    (re.compile(r"\bhave a well-defined\b", _I), "have defined"),
    (re.compile(r"\ba well-defined\b", _I),   "a defined"),
    (re.compile(r"\bof our\b", _I),           "of"),
    (re.compile(r"\ball of our\b", _I),       "all"),
    (re.compile(r"\bfor all of\b", _I),       "for all"),
    (re.compile(r"\beach and every\b", _I),   "each"),
]

# Filler sentence patterns (aggressive: whole lines removed)
_FILLER_SENTENCE_PATTERNS = [
    re.compile(r"^(?:In this (?:section|document|article|guide|chapter|tutorial|post|page),?\s*(?:we will|we'll|we|I will|I'll|you will|you'll)\s*(?:discuss|describe|explain|cover|look at|go over|explore|examine|provide|present|outline|review|walk you through|introduce|show you|take you through)\s+.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:Let's (?:take a look at|look at|examine|explore|dive into|discuss|start by|begin by|talk about|consider|understand|walk through)\s+.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:As we (?:can|have) (?:seen?|noted|discussed|mentioned|explored|covered|learned),?\s*.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:This (?:section|document|guide|article|chapter|tutorial|page|post) (?:provides?|discusses?|describes?|explains?|covers?|presents?|outlines?|reviews?|introduces?|examines?|explores?|walks? (?:you )?through|focuses? on)\s+.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:The (?:following|next|previous) (?:section|sections|chapter|chapters|pages?) .{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:In the following (?:section|sections|chapter|chapters|pages?),?\s*(?:we will|we'll|you will|you'll)\s*.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:Before (?:we|you) (?:dive into|continue|proceed|move on|start|begin)\s*.{0,40},?\s*.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:Throughout this (?:document|guide|section|chapter|tutorial),?\s*.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:(?:At|By) the end of this (?:section|chapter|guide|document),?\s*(?:you will|you'll|we will|we'll)\s*.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:The (?:goal|purpose|aim|objective|intent) of this (?:section|document|guide|chapter|tutorial|article)\s+is\s+to\s+.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:This (?:section|document|guide|chapter|tutorial|article|page)\s+is\s+(?:intended|designed|meant)\s+to\s+.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:Now(?:,)? let's (?:look at|explore|examine|discuss|consider|take a look at|turn to|move on to|talk about)\s+.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:With that (?:out of the way|said|in mind|being said),?\s*.{0,80}\.)\s*$", _IM),
    re.compile(r"^(?:Having (?:covered|discussed|explored|examined|looked at|understood)\s+.{0,40},?\s*(?:let's|we can|we will|now)\s+.{0,60}\.)\s*$", _IM),
    re.compile(r"^(?:Now that (?:we|you) (?:have |)(?:covered|discussed|explored|understand|know|seen|learned)\s*.{0,40},?\s*(?:let's|we can)\s*.{0,60}\.)\s*$", _IM),
]

# Past participles for passive voice compression
_PAST_PARTICIPLES = [
    "stored", "processed", "handled", "managed", "executed", "deployed",
    "built", "validated", "shared", "updated", "deleted", "fetched",
    "logged", "monitored", "encrypted", "authenticated", "authorized",
    "routed", "forwarded", "rejected", "accepted", "mapped", "indexed",
    "serialized", "compressed", "encoded", "decoded", "rotated",
    "enforced", "revoked", "issued", "signed", "verified", "allocated",
    "scaled", "distributed", "replicated", "synchronized", "registered",
    "enabled", "disabled", "activated", "installed", "started", "stopped",
    "initialized", "terminated", "scheduled", "completed", "retried",
    "confirmed", "notified", "published", "consumed", "buffered",
    "committed", "locked", "throttled", "limited", "expired", "refreshed",
    "invalidated", "cleared", "purged", "archived", "migrated", "patched",
    "resolved", "connected", "disconnected", "approved", "granted",
    "checked", "loaded", "mounted", "attached", "bound", "linked",
    "provisioned", "generated", "created", "returned", "sent", "received",
    "called", "triggered", "cached", "queued", "parsed", "exposed",
    "configured", "automated", "tracked", "added", "removed", "used",
    "written", "read", "run", "set", "done", "made",
    "tested", "reviewed", "released", "shipped",
    "maintained", "supported", "protected", "secured",
    "rendered", "compiled", "minified", "bundled",
    "transformed", "converted", "formatted", "normalized", "sanitized",
    "injected", "imported", "exported",
    "subscribed", "unsubscribed",
    "emitted", "dispatched", "reduced", "filtered", "sorted",
    "grouped", "merged", "joined", "split", "chunked",
    "paginated", "streamed", "batched", "flushed",
    "hashed", "salted", "tokenized",
    "deserialized", "marshalled", "unmarshalled",
]

_PASSIVE_VOICE_RE = re.compile(
    r"\b(is|are|was|were) (also |now |then |just |still |already |)?(" + "|".join(_PAST_PARTICIPLES) + r")\b",
    _I,
)

_KEEP_AFTER_THE = frozenset([
    "same", "end", "way", "only", "most", "least", "best", "worst",
    "first", "last", "next", "previous", "rest", "other", "latter", "former",
    "right", "left", "top", "bottom", "above", "below", "inner", "outer",
    "whole", "entire", "very", "fact", "time", "second", "third",
])

_IT_VERBS = (
    r"is|are|was|were|can|could|will|would|should|has|have|had|"
    r"provides?|supports?|handles?|manages?|processes?|creates?|returns?|"
    r"uses?|allows?|enables?|includes?|contains?|requires?|needs?|"
    r"works?|runs?|performs?|executes?|stores?|sends?|receives?|reads?|"
    r"writes?|checks?|validates?|tracks?|monitors?|logs?|caches?|"
    r"generates?|implements?|defines?|represents?|exposes?|accepts?|"
    r"rejects?|connects?|disconnects?|serves?|fetches?|computes?|"
    r"parses?|renders?|converts?|transforms?|extends?|wraps?|encodes?"
)
_IT_VERBS_RE = re.compile(rf"^It ({_IT_VERBS})\b", _M)
_THIS_START_RE = re.compile(r"^This (allows?|enables?|ensures?|means?|helps?|makes?) ", _M)


# ---------------------------------------------------------------------------
# Core Functions
# ---------------------------------------------------------------------------

def _protect_code(text: str) -> tuple[str, list[str], list[str]]:
    code_blocks: list[str] = []
    inline_code: list[str] = []

    def save_block(m: re.Match) -> str:
        code_blocks.append(m.group(0))
        return f"\x00CODEBLOCK{len(code_blocks) - 1}\x00"

    def save_inline(m: re.Match) -> str:
        inline_code.append(m.group(0))
        return f"\x00INLINE{len(inline_code) - 1}\x00"

    # Fenced code blocks
    text = re.sub(r"```[\s\S]*?```", save_block, text)

    # Indented code blocks (2+ lines with 4-space / tab indent)
    def save_indented(m: re.Match) -> str:
        block = m.group(0)
        lines = [l for l in block.split("\n") if l.strip()]
        if len(lines) >= 2:
            code_blocks.append(block)
            return f"\x00CODEBLOCK{len(code_blocks) - 1}\x00"
        return block

    text = re.sub(r"(?:^(?:    |\t).+\n?)+", save_indented, text, flags=_M)

    # Inline code
    text = re.sub(r"`[^`\n]+`", save_inline, text)

    return text, code_blocks, inline_code


def _restore_code(text: str, code_blocks: list[str], inline_code: list[str]) -> str:
    for i, code in enumerate(inline_code):
        text = text.replace(f"\x00INLINE{i}\x00", code)
    for i, block in enumerate(code_blocks):
        text = text.replace(f"\x00CODEBLOCK{i}\x00", block)
    return text


def _strip_comments(text: str) -> str:
    return re.sub(r"<!--[\s\S]*?-->", "", text)


def _strip_decorators(text: str) -> str:
    text = re.sub(r"^\s*[-=*_]{3,}\s*$", "", text, flags=_M)
    text = re.sub(r"^\s*[─━═╌╍╎╏│┃┄┅┆┇┈┉┊┋╭╮╯╰]+\s*$", "", text, flags=_M)
    return text


def _compress_whitespace(text: str, level: str) -> str:
    text = re.sub(r"[ \t]+$", "", text, flags=_M)
    if level == "light":
        text = re.sub(r"\n{4,}", "\n\n\n", text)
    elif level == "medium":
        text = re.sub(r"\n{3,}", "\n\n", text)
    else:
        text = re.sub(r"\n{2,}", "\n", text)
        text = re.sub(r"(^#{1,6}\s.+)\n\n+", r"\1\n", text, flags=_M)
    return text


def _apply_verbose_patterns(text: str, level: str) -> str:
    patterns = list(_VERBOSE_LIGHT)
    if level != "light":
        patterns += _VERBOSE_MEDIUM
    if level == "aggressive":
        patterns += _VERBOSE_AGGRESSIVE + _VERBOSE_ULTRA

    for pattern, replacement in patterns:
        text = pattern.sub(replacement, text)

    text = re.sub(r" {2,}", " ", text)
    text = re.sub(r"\. \.", ".", text)
    text = re.sub(r", ,", ",", text)
    text = re.sub(r"\(\s*\)", "", text)
    text = re.sub(r"^ ([A-Z])", r"\1", text, flags=_M)
    return text


def _compress_passive_voice(text: str, level: str) -> str:
    if level != "aggressive":
        return text

    def _replace(m: re.Match) -> str:
        adv = (m.group(2) or "").strip()
        pp = m.group(3)
        return f"{adv} {pp}" if adv else pp

    return _PASSIVE_VOICE_RE.sub(_replace, text)


def _strip_articles_broad(text: str, level: str) -> str:
    if level != "aggressive":
        return text

    def _the_prep(m: re.Match) -> str:
        prep, noun = m.group(1), m.group(2)
        if noun.lower() in _KEEP_AFTER_THE:
            return m.group(0)
        return f"{prep} {noun}"

    text = re.sub(
        r"\b(at|in|of|for|by|to|from|via|through|with|per|across|between|within|into|onto|upon|under|over|after|before|during|along|against|about) the ([a-z][a-zA-Z0-9]*)\b",
        _the_prep,
        text,
    )
    text = re.sub(r"\bthe ([A-Z]{2,})\b", r"\1", text)
    text = re.sub(r"\bthe ([A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b", r"\1", text)
    text = re.sub(r"^The ([a-z])", lambda m: m.group(1).upper(), text, flags=_M)
    text = _IT_VERBS_RE.sub(r"\1", text)
    text = _THIS_START_RE.sub(
        lambda m: m.group(1)[0].upper() + m.group(1)[1:] + " ", text
    )
    return text


def _compress_relative_clauses(text: str, level: str) -> str:
    if level == "light":
        return text

    _subordinating = re.compile(
        r"\b(that|which|who|when|where|because|if|while|although|unless)\b"
    )

    def _which_is(m: re.Match) -> str:
        desc = m.group(3).strip()
        if _subordinating.search(desc):
            return m.group(0)
        return f" ({desc}){m.group(4)}"

    text = re.sub(
        r",\s*which (is|are) (a |an |the )?([a-z][a-z\s\-]{2,40}?)(,|\.|;|$)",
        _which_is,
        text,
        flags=_M,
    )

    if level == "aggressive":
        text = re.sub(r",\s*which (can|could|will|would|should|must|may|might)\b", r" that \1", text)
        text = re.sub(r",\s*which (has|have|had)\b", r" that \1", text)
        text = re.sub(r",\s*which ([a-z]{3,}s)\b", r" that \1", text)
        text = re.sub(
            r"\bthat (is|are) (a |an )?(critical|important|essential|key|main|primary|core|central|major|minor|optional|required|necessary|useful|effective|efficient|available|common|popular|standard|typical|default|custom|external|internal|local|global|public|private|shared|dedicated|configurable)\b",
            r"\3",
            text,
            flags=_I,
        )
    return text


def _compress_headings(text: str, level: str) -> str:
    if level == "light":
        return text
    text = re.sub(r"^(#{1,6} )Introduction to (.+)$", r"\1\2", text, flags=_IM)
    text = re.sub(r"^(#{1,6} )Overview of (.+)$", r"\1\2 Overview", text, flags=_IM)
    text = re.sub(r"^(#{1,6} )Getting Started with (.+)$", r"\1\2 Setup", text, flags=_IM)
    if level == "aggressive":
        text = re.sub(r"^(#{1,6} .+?)\s+Section\s*$", r"\1", text, flags=_IM)
        text = re.sub(r"^(#{1,6} )(?:Comprehensive|Complete|Detailed|Basic|Simple|Advanced) ", r"\1", text, flags=_IM)
        text = re.sub(r"^#{5,}\s", "#### ", text, flags=_M)
    return text


def _strip_formatting(text: str, level: str) -> str:
    text = re.sub(r"^(#{1,6})\s+\*\*(.+?)\*\*\s*$", r"\1 \2", text, flags=_M)
    text = re.sub(r"^(#{1,6})\s+__(.+?)__\s*$", r"\1 \2", text, flags=_M)
    if level in ("medium", "aggressive"):
        text = re.sub(r"^(#{1,6})\s+[_*](.+?)[_*]\s*$", r"\1 \2", text, flags=_M)
        text = re.sub(r"^(#{1,6}\s.+?)\s*#+\s*$", r"\1", text, flags=_M)
        text = re.sub(r"^(\s*[-*+]\s+)\*\*(.+?)\*\*(\s*[-–:])", r"\1\2\3", text, flags=_M)
    if level == "aggressive":
        text = re.sub(r"^#{5,}\s", "#### ", text, flags=_M)
        text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)
        text = re.sub(r"__(.+?)__", r"\1", text)
        text = re.sub(r"(?<!\*)\*([^*\n]+?)\*(?!\*)", r"\1", text)
        text = re.sub(r"(?<!_)_([^_\n]+?)_(?!_)", r"\1", text)
    return text


def _compress_tables(text: str, level: str) -> str:
    def _process_table(m: re.Match) -> str:
        header_row = m.group(1)
        body_rows_str = m.group(3)

        def parse_row(row: str) -> list[str]:
            return [c.strip() for c in row.split("|")[1:-1]]

        headers = parse_row(header_row)
        rows = [parse_row(r) for r in body_rows_str.strip().split("\n")]

        if level == "light":
            min_sep = "|" + "|".join("-" * max(1, len(h)) for h in headers) + "|"
            min_header = "|" + "|".join(headers) + "|"
            min_body = "\n".join("|" + "|".join(r) + "|" for r in rows)
            return f"{min_header}\n{min_sep}\n{min_body}"

        if level == "medium":
            header_line = "|".join(headers)
            body_lines = "\n".join("|".join(r) for r in rows)
            return f"[{header_line}]\n{body_lines}"

        # aggressive: key:value inline
        return "\n".join(
            "|".join(f"{headers[i]}:{cell}" for i, cell in enumerate(row))
            for row in rows
        )

    return re.sub(
        r"(\|.+\|)\n(\|[\s:|-]+\|)\n((?:\|.+\|\n?)+)",
        _process_table,
        text,
    )


def _compress_lists(text: str, level: str) -> str:
    if level == "light":
        return text

    if level in ("medium", "aggressive"):
        text = re.sub(r"^(\s*[-*+]\s+)\*\*(.+?)\*\*(\s*[-–:]?\s*)", r"\1\2\3", text, flags=_M)
        text = re.sub(r"^(\s*\d+\.\s+)\*\*(.+?)\*\*(\s*[-–:]?\s*)", r"\1\2\3", text, flags=_M)

    if level == "aggressive":
        lines = text.split("\n")
        result: list[str] = []
        list_buffer: list[str] = []
        in_list = False
        list_prefix = "-"

        def flush_list() -> None:
            nonlocal in_list
            if len(list_buffer) >= 3:
                result.append("; ".join(list_buffer))
            else:
                result.extend(f"{list_prefix} {item}" for item in list_buffer)
            list_buffer.clear()
            in_list = False

        for line in lines:
            bullet_match = re.match(r"^(\s*)([-*+]|\d+\.)\s+(.+)", line)
            if bullet_match:
                indent = bullet_match.group(1)
                marker = bullet_match.group(2)
                content = bullet_match.group(3).strip()
                plain_content = re.sub(r"\*\*(.+?)\*\*", r"\1", content)
                is_simple = len(plain_content) < 100 and "`" not in content

                if is_simple and len(indent) == 0:
                    if not in_list:
                        list_prefix = "-" if marker in ("-", "*", "+") else "1."
                        in_list = True
                    content = re.sub(r"\*\*(.+?)\*\*", r"\1", content)
                    list_buffer.append(content)
                    continue

            if in_list and (not bullet_match or len(bullet_match.group(1)) == 0):
                flush_list()

            result.append(line)

        if in_list:
            flush_list()

        text = "\n".join(result)

        # Numbered list compression (4+ items)
        def compress_numbered(m: re.Match) -> str:
            items = []
            for line in m.group(0).strip().split("\n"):
                nm = re.match(r"^\d+\.\s+(.+)", line)
                items.append(nm.group(1).strip() if nm else line)
            return " | ".join(f"{i+1}) {item}" for i, item in enumerate(items)) + "\n"

        text = re.sub(r"(?:^\d+\.\s+.{1,60}\n){4,}", compress_numbered, text, flags=_M)

    return text


def _clean_links(text: str, level: str) -> str:
    text = re.sub(r'\[([^\]]+)\]\(([^)]+)\s+""\)', r"[\1](\2)", text)
    if level in ("medium", "aggressive"):
        text = re.sub(r'\[([^\]]+)\]\(([^")]+)\s+"[^"]*"\)', r"[\1](\2)", text)
    if level == "aggressive":
        text = re.sub(r"<(https?://[^>]+)>", r"\1", text)
        text = re.sub(r"\[([^\]]+)\]\(\1\)", r"\1", text)
        # Remove unused reference definitions
        ref_defs: dict[str, str] = {}
        for m in re.finditer(r"^\[([^\]]+)\]:\s*(.+)$", text, _M):
            ref_defs[m.group(1).lower()] = m.group(0)
        for label, full_match in ref_defs.items():
            escaped = re.escape(label)
            if not re.search(rf"\[([^\]]+)\]\[{escaped}\]", text, _I):
                text = text.replace(full_match, "")
    return text


def _compress_images(text: str, level: str) -> str:
    if level == "light":
        return text
    text = re.sub(r'!\[([^\]]*)\]\(([^")]+)\s+"[^"]*"\)', r"![\1](\2)", text)
    if level == "aggressive":
        def shorten_alt(m: re.Match) -> str:
            alt = m.group(1)
            short = alt.split(r"[.,;]")[0].strip()[:40]
            return f"![{short}]"
        text = re.sub(r"!\[([^\]]{50,})\]", shorten_alt, text)
    return text


def _deduplicate_repeats(text: str, min_occurrences: int = 3) -> tuple[str, dict | None]:
    MIN_LEN = 12
    phrases: dict[str, int] = {}
    words = text.split()

    for length in range(3, 11):
        for i in range(len(words) - length + 1):
            phrase = " ".join(words[i:i + length])
            if len(phrase) >= MIN_LEN and "\x00" not in phrase and not phrase.startswith("#"):
                phrases[phrase] = phrases.get(phrase, 0) + 1

    candidates = sorted(
        ({"phrase": p, "count": c, "savings": (len(p) - 3) * c}
         for p, c in phrases.items() if c >= min_occurrences),
        key=lambda x: -x["savings"],
    )[:25]

    if not candidates:
        return text, None

    dictionary: dict[str, str] = {}
    result = text
    sym_index = 1

    for item in candidates:
        phrase = item["phrase"]
        symbol = f"§{sym_index}"
        regex = re.compile(re.escape(phrase))
        actual_count = len(regex.findall(result))

        if actual_count >= min_occurrences:
            dict_entry = f"{symbol}={phrase}"
            savings = (len(phrase) * actual_count) - (len(symbol) * actual_count + len(dict_entry) + 1)
            if savings > 8:
                result = regex.sub(symbol, result)
                dictionary[symbol] = phrase
                sym_index += 1

    return result, dictionary if dictionary else None


def _remove_filler_sentences(text: str) -> str:
    for pattern in _FILLER_SENTENCE_PATTERNS:
        text = pattern.sub("", text)
    return text


def _cleanup_artifacts(text: str) -> str:
    text = re.sub(r"^\s+$", "", text, flags=_M)
    text = re.sub(r"^\s*[-*+]\s*$", "", text, flags=_M)
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = text.strip()
    text = re.sub(r"\.\s+([a-z])", lambda m: f". {m.group(1).upper()}", text)
    text = re.sub(r"^([a-z])", lambda m: m.group(1).upper(), text, flags=_M)
    text = re.sub(r" {2,}", " ", text)
    text = re.sub(r" ([.,;:!?])", r"\1", text)
    text = re.sub(r"\(\s*\)", "", text)
    text = re.sub(r"\[\s*\]", "", text)
    text = re.sub(r"\.{2,}", ".", text)
    text = re.sub(r",,+", ",", text)
    text = re.sub(r",\.", ".", text)
    text = re.sub(r"\.\s*,", ".", text)
    text = re.sub(r",\s*\.", ".", text)
    text = re.sub(r"\s+/\s+", "/", text)
    text = re.sub(r"\.\s*To\s+To\b", ". To", text)
    text = re.sub(r"^:\s*", "", text, flags=_M)
    text = re.sub(r"\b(\w{4,}) \1\b", r"\1", text)
    return text


# ---------------------------------------------------------------------------
# Token Estimation
# ---------------------------------------------------------------------------

def estimate_tokens(text: str) -> int:
    """Estimate BPE token count for text. Fast heuristic, ±10% accuracy."""
    if not text:
        return 0

    tokens = 0.0
    chunks = re.findall(r"\S+|\s+", text)

    for chunk in chunks:
        if re.match(r"^\s+$", chunk):
            newlines = chunk.count("\n")
            tokens += newlines
            if " " in chunk or "\t" in chunk:
                tokens += 0.25
        else:
            length = len(chunk)
            if length <= 6:
                tokens += 1
            elif length <= 10:
                tokens += 2
            elif length <= 15:
                tokens += 3
            else:
                tokens += length / 4

            special_chars = len(re.findall(r"[^a-zA-Z0-9]", chunk))
            tokens += special_chars * 0.3

    return round(tokens)


# ---------------------------------------------------------------------------
# Public API
# ---------------------------------------------------------------------------

@dataclass
class CompressionStats:
    input_tokens: int
    output_tokens: int
    saved: int
    pct: float
    input_chars: int
    output_chars: int
    level: str
    dictionary: int  # number of deduplicated phrases


@dataclass
class CompressResult:
    output: str
    stats: CompressionStats


_VALID_LEVELS = frozenset(["light", "medium", "aggressive"])

_DEFAULT_RULES = {
    "blank_lines": True,
    "tables": True,
    "verbose": True,
    "formatting": True,
    "decorators": True,
    "dedup": True,
    "comments": True,
    "links": True,
    "lists": True,
    "images": True,
    "relative_clauses": True,
    "headings": True,
    "passive_voice": True,
    "article_stripping": True,
    "token_synonyms": True,
}


def compress(
    text: str,
    level: Literal["light", "medium", "aggressive"] = "medium",
    rules: dict | None = None,
) -> CompressResult:
    """
    Compress markdown text to reduce LLM token usage.

    Args:
        text:  Input markdown or plain text.
        level: Compression level — 'light', 'medium', or 'aggressive'.
        rules: Optional dict to override individual rule toggles.

    Returns:
        CompressResult with .output (str) and .stats (CompressionStats).
    """
    if not text or not text.strip():
        return CompressResult(
            output="",
            stats=CompressionStats(0, 0, 0, 0.0, 0, 0, level, 0),
        )

    if level not in _VALID_LEVELS:
        raise ValueError(f"level must be one of {sorted(_VALID_LEVELS)}, got {level!r}")

    r = {**_DEFAULT_RULES, **(rules or {})}
    input_tokens = estimate_tokens(text)
    input_chars = len(text)

    protected, code_blocks, inline_code = _protect_code(text)

    # Compress blank lines inside code blocks
    if level == "aggressive":
        code_blocks = [re.sub(r"\n\n+", "\n", b) for b in code_blocks]
    elif level == "medium":
        code_blocks = [re.sub(r"\n{3,}", "\n\n", b) for b in code_blocks]

    t = protected
    if r["comments"]:          t = _strip_comments(t)
    if r["decorators"]:        t = _strip_decorators(t)
    if r["verbose"] and level == "aggressive":
        t = _remove_filler_sentences(t)
    if r["verbose"]:           t = _apply_verbose_patterns(t, level)
    if r["token_synonyms"]:    t, _ = apply_token_synonyms(t, level)
    if r["relative_clauses"]:  t = _compress_relative_clauses(t, level)
    if r["passive_voice"]:     t = _compress_passive_voice(t, level)
    if r["article_stripping"]: t = _strip_articles_broad(t, level)
    if r["headings"]:          t = _compress_headings(t, level)
    if r["formatting"]:        t = _strip_formatting(t, level)
    if r["tables"]:            t = _compress_tables(t, level)
    if r["lists"]:             t = _compress_lists(t, level)
    if r["links"]:             t = _clean_links(t, level)
    if r["images"]:            t = _compress_images(t, level)
    if r["blank_lines"]:       t = _compress_whitespace(t, level)

    dictionary: dict | None = None
    if r["dedup"] and level != "light" and len(t) > 400:
        min_occ = 2 if level == "aggressive" else 3
        t, dictionary = _deduplicate_repeats(t, min_occ)

    t = _cleanup_artifacts(t)
    t = _restore_code(t, code_blocks, inline_code)

    if dictionary:
        dict_line = "|".join(f"{sym}={phrase}" for sym, phrase in dictionary.items())
        t = f"[dict:{dict_line}]\n{t}"

    output_tokens = estimate_tokens(t)
    saved = input_tokens - output_tokens
    pct = round((saved / input_tokens) * 1000) / 10 if input_tokens > 0 else 0.0

    return CompressResult(
        output=t,
        stats=CompressionStats(
            input_tokens=input_tokens,
            output_tokens=output_tokens,
            saved=saved,
            pct=pct,
            input_chars=input_chars,
            output_chars=len(t),
            level=level,
            dictionary=len(dictionary) if dictionary else 0,
        ),
    )
