# AUTO-GENERATED FILE - DO NOT EDIT
# This file was automatically generated by the XDK build tool.
# Any manual changes will be overwritten on the next generation.
#!/usr/bin/env python3
"""
AUTO-GENERATED FILE - DO NOT EDIT
This file was automatically generated by the XDK build tool.
Any manual changes will be overwritten on the next generation.

Process Sphinx-generated markdown documentation for Mintlify.
"""

import os
import sys
import json
import re
import shutil
from pathlib import Path
from typing import Dict, List, Optional, Set

print("🚀 Processing X API SDK Documentation for Mintlify...")

# Mintlify configuration
MINTLIFY_CONFIG = {
    "outputDir": "mintlify-docs",
    "baseUrl": "https://docs.x.com",
    "title": "X API SDK v0.2.3-beta",
    "description": "Python SDK for the X API with comprehensive pagination, authentication, and streaming support.",
    "version": "0.2.3-beta",
    "githubUrl": "https://github.com/xdevplatform/xdk",
}


def clean_title(title: str, file_path: str = "", content: str = "") -> str:
    """Clean and improve title formatting."""
    if not isinstance(title, str):
        title = str(title)
    # Try to extract class name from content first (more reliable)
    if content:
        # Look for class definition: ### *class* xdk.module.ClassName
        class_match = re.search(
            r"\*class\*\s+[^\s]*\.([A-Z][a-zA-Z0-9_]*Client|Client|Paginator|OAuth2PKCEAuth)\b",
            content,
        )
        if class_match:
            return class_match.group(1)
    # Remove module suffix
    title = re.sub(r"\s+module\s*$", "", title, flags=re.IGNORECASE)
    # Extract class name from patterns like "xdk.users.client.UsersClient"
    class_match = re.search(
        r"\.([A-Z][a-zA-Z0-9_]+Client|Paginator|OAuth2PKCEAuth)\b", title
    )
    if class_match:
        return class_match.group(1)
    # Fallback to generic Client only if no specific client found
    if "Client" in title and not re.search(r"[A-Z][a-zA-Z0-9_]+Client", title):
        class_match = re.search(r"\.(Client)\b", title)
        if class_match:
            return class_match.group(1)
    # Extract class name from patterns like "*class* xdk.users.client.UsersClient"
    class_match2 = re.search(
        r"\*class\*\s+[^\s]*\.([A-Z][a-zA-Z0-9_]*Client|Client|Paginator|OAuth2PKCEAuth)\b",
        title,
    )
    if class_match2:
        return class_match2.group(1)
    # Remove xdk. prefix
    title = re.sub(r"^xdk\.", "", title)
    # Convert snake_case to PascalCase for module names
    if "." in title and not title[0].isupper():
        parts = title.split(".")
        # Capitalize each part
        title = ".".join(p.capitalize() for p in parts)
    # Clean up
    title = (
        re.sub(r"<.*?>", "", title)  # Remove generic type parameters
        .replace("Class: ", "")
        .replace("Interface: ", "")
        .replace("*class*", "")
        .replace("*", "")
        .replace("\\", "")
        .replace("\n", " ")
        .strip()
    )
    return title


def generate_frontmatter(
    title: str, sidebar_title: Optional[str] = None, file_path: str = ""
) -> str:
    """Generate Mintlify frontmatter."""
    cleaned_title = clean_title(title, file_path)
    cleaned_sidebar = (
        clean_title(sidebar_title, file_path) if sidebar_title else cleaned_title
    )
    frontmatter = f'title: "{cleaned_title}"\n'
    if sidebar_title:
        frontmatter += f'sidebarTitle: "{cleaned_sidebar}"\n'
    return f"---\n{frontmatter}---\n\n"


def reorganize_class_structure(content: str, file_path: str) -> str:
    """Reorganize content into proper sections like TypeScript."""
    # Check if this is a class/client file
    if "client" not in file_path.lower() and "models" not in file_path.lower():
        return content
    # Find class definition - handle both formats
    # Pattern 1: ### *class* xdk.module.ClassName(params)
    # Pattern 2: ### `*class* xdk.module.ClassName`(params)
    class_match = re.search(
        r"###\s+(?:`?)?\*class\*\s+([^\n(]+)(?:`?)?\s*\(([^)]*)\)", content
    )
    if not class_match:
        return content
    class_name_full = class_match.group(1).strip().replace("*", "").replace("`", "")
    # Extract just the class name (last part after last dot)
    class_name = (
        class_name_full.split(".")[-1] if "." in class_name_full else class_name_full
    )
    class_params = class_match.group(2).strip()
    # Extract description after class definition
    class_start = class_match.end()
    next_section = content.find("###", class_start)
    if next_section == -1:
        next_section = len(content)
    description = content[class_start:next_section].strip()
    # Extract "Bases:" line for later use as Badge
    bases_match = re.search(r"Bases:\s*`([^`]+)`", description, re.IGNORECASE)
    bases = bases_match.group(1).strip() if bases_match else None
    # Remove "Bases:" line - we'll add it as a Badge
    description = re.sub(
        r"Bases:\s*`[^`]+`\s*\n?", "", description, flags=re.IGNORECASE
    )
    # Remove any stray closing parentheses
    description = re.sub(r"^\)\s*\n?", "", description)
    description = description.strip()
    # Find all methods and properties
    methods = []
    constructors = []
    properties = []
    # Pattern for methods: ### method_name(params) → ReturnType
    method_pattern = r"###\s+`?([^\n(]+)`?\s*\(([^)]*)\)(?:\s*→\s*([^\n]+))?"
    for match in re.finditer(method_pattern, content):
        method_name = (
            match.group(1).strip().replace("`", "").replace("\\", "").replace("*", "")
        )
        params = match.group(2).strip()
        return_type = match.group(3).strip() if match.group(3) else None
        # Find method body
        method_start = match.start()
        next_method = content.find("###", method_start + 1)
        if next_method == -1:
            method_body = content[method_start:]
        else:
            method_body = content[method_start:next_method]
        method_info = {
            "name": method_name,
            "params": params,
            "return_type": return_type,
            "body": method_body,
        }
        if method_name == "__init__" or "constructor" in method_name.lower():
            constructors.append(method_info)
        elif method_name.startswith("property") or "property" in method_body.lower():
            properties.append(method_info)
        else:
            methods.append(method_info)
    # Rebuild content with proper sections
    sections = []
    # Class definition and description
    sections.append(f"## {class_name}\n\n")
    sections.append('<Badge color="blue">Class</Badge>\n')
    if bases:
        sections.append(f'\n<Badge color="gray">Bases: {bases}</Badge>\n')
    if description:
        sections.append(f"\n{description}\n")
    # Constructors section
    if constructors:
        sections.append("\n## Constructors\n")
        for const in constructors:
            sections.append(const["body"])
    # Methods section
    if methods:
        sections.append("\n## Methods\n")
        for method in methods:
            sections.append(method["body"])
    # Properties section
    if properties:
        sections.append("\n## Properties\n")
        for prop in properties:
            sections.append(prop["body"])
    # If we found methods/constructors, rebuild content
    if constructors or methods or properties:
        before_class = content[: class_match.start()]
        # Get remaining content after last method
        if methods:
            last_method_end = content.rfind(methods[-1]["body"])
            remaining = content[last_method_end + len(methods[-1]["body"]) :]
        elif constructors:
            last_constructor_end = content.rfind(constructors[-1]["body"])
            remaining = content[last_constructor_end + len(constructors[-1]["body"]) :]
        else:
            remaining = content[next_section:]
        # Clean up any stray characters and duplicate class definitions
        remaining = re.sub(r"^\)\s*\n", "", remaining)
        # Remove duplicate class definitions that might have been left behind
        # Pattern: ### `class xdk.module.ClassName` followed by description and parameters
        # Match the full duplicate class definition block
        remaining = re.sub(
            r"###\s+`?class\s+xdk\.[^\n]+\n\n[^\n]+\n\n(?:####\s+Parameters[^\n]+\n\n<ParamField[^>]+>\s*</ParamField>\s*\n)?",
            "",
            remaining,
        )
        # Also remove any standalone class definitions without parameters
        remaining = re.sub(
            r"###\s+`?\*?class\*?\s+xdk\.[^\n]+\n\n[^\n]+\n\n", "", remaining
        )
        # Remove any remaining class definition patterns (more aggressive)
        remaining = re.sub(
            r"###\s+`?class\s+[^\n]+\n\n[^\n]+\n\n(?:####\s+Parameters[^\n]+\n\n)?",
            "",
            remaining,
        )
        return before_class + "\n".join(sections) + remaining
    return content


def clean_class_definitions(content: str) -> str:
    """Clean up class definition headers."""
    # Remove any remaining class definition headers that weren't processed
    # Pattern: ### `*class* xdk.module.ClassName`(params) or ## xdk.module.ClassName
    # These should already be converted by reorganize_class_structure, but clean up any leftovers
    # Remove any stray class definition patterns
    content = re.sub(r"##\s+xdk\.[^\n]+\n\n\)\s*\n", "", content)
    # Clean up any remaining "Bases:" lines that weren't converted
    content = re.sub(r"^Bases:\s*`[^`]+`\s*\n?", "", content, flags=re.MULTILINE)
    return content


def fix_method_names(content: str) -> str:
    """Fix method names with escaped underscores."""
    # Pattern: ### ``\_\_init_\_`` -> ### `__init__`
    # Handle double backticks with escaped underscores
    content = re.sub(r"###\s+``\\?(_+[^`]+_+)``", r"### `\1`", content)
    # Pattern: ### ``method\_name`` -> ### `method_name`
    content = re.sub(r"###\s+``([^`]*)\\?(_[^`]*)``", r"### `\1\2`", content)
    # Pattern: ### ``\_\_init_\_``(params) -> ### `__init__`(params)
    content = re.sub(r"###\s+``([^`]*)\\?(_[^`]*)``\s*\(", r"### `\1\2`(", content)
    # Remove any remaining escaped underscores in method names (single backticks)
    content = re.sub(r"###\s+`([^`]*)\\?(_[^`]*)`\s*\(", r"### `\1\2`(", content)
    # Fix any remaining escaped underscores in code blocks
    content = re.sub(r"`([^`]*)\\?(_[^`]*)`", r"`\1\2`", content)
    return content


def improve_method_formatting(content: str) -> str:
    """Improve method formatting to match TypeScript style with ParamField components."""
    # Pattern: ### method_name(params) → ReturnType
    def convert_method(match):
        method_header = match.group(0)
        method_name = (
            match.group(1)
            .strip()
            .replace("*", "")
            .replace("`", "")
            .replace("\\", "")
            .replace("_", "_")
        )
        params_str = match.group(2).strip() if match.group(2) else ""
        return_type = match.group(3).strip() if match.group(3) else None
        # Find method body (until next ### or ####)
        method_start = match.start()
        next_method = content.find("###", method_start + 1)
        if next_method == -1:
            method_body = content[method_start:]
        else:
            method_body = content[method_start:next_method]
        # Extract description (text after method header, before :param)
        desc_match = re.search(
            r"###[^\n]+\n\n([^\n]+(?:\n(?!:param|####|###|####)[^\n]+)*)",
            method_body,
            re.MULTILINE,
        )
        description = desc_match.group(1).strip() if desc_match else ""
        # Remove method name from description if it appears
        description = re.sub(
            r"^" + re.escape(method_name) + r"\s*$", "", description, flags=re.MULTILINE
        ).strip()
        # Parse parameters and convert to ParamField components
        param_fields = []
        if params_str:
            # Simple parameter parsing (split by comma, but handle type annotations)
            params = []
            current = ""
            depth = 0
            for char in params_str:
                if char in "[(":
                    depth += 1
                elif char in "])":
                    depth -= 1
                elif char == "," and depth == 0:
                    if current.strip():
                        params.append(current.strip())
                    current = ""
                    continue
                current += char
            if current.strip():
                params.append(current.strip())
            for param in params:
                # Parse: name: type = default
                # Handle cases like: client: [Client](xdk.md#xdk.Client)
                param_match = re.match(
                    r"(\w+)(?:\s*:\s*([^=]+))?(?:\s*=\s*(.+))?$", param.strip()
                )
                if param_match:
                    param_name = param_match.group(1)
                    param_type_raw = (
                        param_match.group(2).strip() if param_match.group(2) else "Any"
                    )
                    param_default = (
                        param_match.group(3).strip() if param_match.group(3) else None
                    )
                    # Clean type - handle markdown links first
                    # Pattern: [Type](path#anchor) -> extract just "Type"
                    link_match = re.search(r"\[([^\]]+)\]\(([^\)]+)\)", param_type_raw)
                    if link_match:
                        param_type = link_match.group(1)  # Use just the link text
                    else:
                        # No link, use the raw type but clean it
                        param_type = param_type_raw
                        # Remove file references and anchors
                        param_type = re.sub(r"[a-z_]+\.(md|py)#[^\s]*", "", param_type)
                        param_type = re.sub(r"#[^\s]*", "", param_type)
                        # Remove incomplete link patterns like "Client]("
                        param_type = re.sub(r"\]\([^\)]*$", "", param_type)
                        param_type = re.sub(r"\]\([^\)]*\)", "", param_type)
                    # Clean up the type string
                    param_type = param_type.replace("|", " or ").strip()
                    # Remove any trailing/leading brackets and parentheses
                    param_type = re.sub(r"^[\[\(]+", "", param_type)
                    param_type = re.sub(r"[\]\)]+$", "", param_type)
                    # Remove any remaining incomplete patterns
                    param_type = re.sub(r"\]\(.*$", "", param_type)
                    # Escape angle brackets for MDX
                    param_type = param_type.replace("<", "&lt;").replace(">", "&gt;")
                    # Clean up extra spaces
                    param_type = re.sub(r"\s+", " ", param_type).strip()
                    # If type is empty or just brackets, default to Any
                    if not param_type or param_type in ["[", "]", "()", "(", ")", "]("]:
                        param_type = "Any"
                    # Find param description
                    param_desc_match = re.search(
                        rf":param\s+{param_name}:\s*([^\n]+)", method_body
                    )
                    param_desc = (
                        param_desc_match.group(1).strip() if param_desc_match else ""
                    )
                    # Build ParamField - use path instead of name
                    # For Python methods, parameters are function arguments
                    # Use "path" location for most parameters, or "body" if it's clearly a request body
                    param_location = (
                        "body"
                        if param_name.lower() in ["body", "data", "payload", "request"]
                        else "path"
                    )
                    param_field = f'<ParamField path="{param_location}.{param_name}" type="{param_type}"'
                    if param_default:
                        # Escape default value
                        param_default_clean = (
                            param_default.replace('"', "&quot;")
                            .replace("<", "&lt;")
                            .replace(">", "&gt;")
                        )
                        param_field += f' default="{param_default_clean}"'
                    param_field += ">"
                    if param_desc:
                        param_field += f"\n{param_desc}\n"
                    param_field += "</ParamField>"
                    param_fields.append(param_field)
        # Build new method format
        new_method = f"### `{method_name}`\n\n"
        if description:
            new_method += f"{description}\n\n"
        if param_fields:
            new_method += "#### Parameters\n\n"
            new_method += "\n\n".join(param_fields) + "\n\n"
        if return_type:
            # Extract return description
            return_desc_match = re.search(r":param\s+Returns?:\s*([^\n]+)", method_body)
            return_desc = (
                return_desc_match.group(1).strip() if return_desc_match else ""
            )
            # Clean return type - handle markdown links
            return_type_clean = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", return_type)
            return_type_clean = re.sub(
                r"[a-z_]+\.(md|py)#[^\s]+", "", return_type_clean
            )
            return_type_clean = return_type_clean.replace("[", "").replace("]", "")
            return_type_clean = return_type_clean.replace("<", "&lt;").replace(
                ">", "&gt;"
            )
            return_type_clean = re.sub(
                r":param\s+\w+:\s*", "", return_type_clean
            ).strip()
            return_type_clean = re.sub(r"\s+", " ", return_type_clean).strip()
            new_method += "#### Returns\n\n"
            new_method += f"`{return_type_clean}`"
            if return_desc and return_desc != return_type_clean:
                new_method += f" - {return_desc}"
            new_method += "\n\n"
        return new_method
    # Convert method signatures to use ParamField components
    # Match: ### method_name(params) → ReturnType
    def process_all_methods(text):
        # Find all method definitions
        method_pattern = r"###\s+`?([^\n(]+)`?\s*\(([^)]*)\)(?:\s*→\s*([^\n]+))?"
        methods = list(re.finditer(method_pattern, text, re.MULTILINE))
        if not methods:
            return text
        # Build result by processing each method and replacing its entire section
        result_parts = []
        last_pos = 0
        for i, match in enumerate(methods):
            # Add content before this method
            result_parts.append(text[last_pos : match.start()])
            # Find where this method's body ends (next method or end of content)
            method_start = match.start()
            if i + 1 < len(methods):
                next_method_start = methods[i + 1].start()
            else:
                next_method_start = len(text)
            # Get the full method section to replace
            method_section = text[method_start:next_method_start]
            # Convert this method
            converted = convert_method(match)
            # Remove the old method content from the section
            result_parts.append(converted)
            last_pos = next_method_start
        # Add remaining content
        result_parts.append(text[last_pos:])
        return "".join(result_parts)
    content = process_all_methods(content)
    # Clean up any remaining :param lines that weren't converted
    content = re.sub(r":param\s+(\w+):\s*([^\n]+)", r"**`\1`** - \2", content)
    content = re.sub(r":param\s+Returns?:\s*([^\n]+)", r"**Returns:** \1", content)
    # Remove duplicate return descriptions
    content = re.sub(
        r"(#### Returns\n\n`[^\n]+`[^\n]+\n\n)(\*\*`[^\n]+\*\*[^\n]+\n)+",
        r"\1",
        content,
    )
    # Remove duplicate method descriptions
    content = re.sub(r"(### `[^\n]+`\n\n[^\n]+\n\n[^\n]+\n\n)(\1)", r"\1", content)
    # Clean up any remaining old format parameter lines after ParamField sections
    content = re.sub(r"(</ParamField>\n\n)(\*\*`[^\n]+\*\*[^\n]+\n)+", r"\1", content)
    # Remove duplicate "Returns:" text in return sections
    content = re.sub(
        r"(#### Returns\n\n`[^\n]+`)\s*-\s*\*\*`[^\n]+\*\*\s*-\s*([^\n]+)",
        r"\1 - \2",
        content,
    )
    # Remove duplicate class definitions that appear after the main class header
    # Pattern: ### `class xdk.module.ClassName` followed by description and parameters
    # Find the main class header (should be ## ClassName)
    main_class_match = re.search(
        r"##\s+([A-Z][a-zA-Z0-9_]+Client|Client|Paginator|OAuth2PKCEAuth|BaseModel)",
        content,
    )
    if main_class_match:
        # Everything after the main class header should not have duplicate class definitions
        before_main = content[: main_class_match.end()]
        after_main = content[main_class_match.end() :]
        # Remove any class definitions from after_main
        after_main = re.sub(
            r"###\s+`?class\s+xdk\.[^\n]+\n\n[^\n]+\n\n(?:####\s+Parameters[^\n]+\n\n<ParamField[^>]+>\s*</ParamField>\s*\n)?",
            "",
            after_main,
        )
        content = before_main + after_main
    # Remove duplicate parameter sections (#### Parameters appearing twice in a row)
    content = re.sub(
        r"(#### Parameters\n\n<ParamField[^>]+>\s*</ParamField>\s*\n)\1", r"\1", content
    )
    return content


    def convert_method(match):
        method_header = match.group(0)
        method_name = (
            match.group(1)
            .strip()
            .replace("*", "")
            .replace("`", "")
            .replace("\\", "")
            .replace("_", "_")
        )
        params_str = match.group(2).strip() if match.group(2) else ""
        return_type = match.group(3).strip() if match.group(3) else None
        # Find method body (until next ### or ####)
        method_start = match.start()
        next_method = content.find("###", method_start + 1)
        if next_method == -1:
            method_body = content[method_start:]
        else:
            method_body = content[method_start:next_method]
        # Extract description (text after method header, before :param)
        desc_match = re.search(
            r"###[^\n]+\n\n([^\n]+(?:\n(?!:param|####|###|####)[^\n]+)*)",
            method_body,
            re.MULTILINE,
        )
        description = desc_match.group(1).strip() if desc_match else ""
        # Remove method name from description if it appears
        description = re.sub(
            r"^" + re.escape(method_name) + r"\s*$", "", description, flags=re.MULTILINE
        ).strip()
        # Parse parameters and convert to ParamField components
        param_fields = []
        if params_str:
            # Simple parameter parsing (split by comma, but handle type annotations)
            params = []
            current = ""
            depth = 0
            for char in params_str:
                if char in "[(":
                    depth += 1
                elif char in "])":
                    depth -= 1
                elif char == "," and depth == 0:
                    if current.strip():
                        params.append(current.strip())
                    current = ""
                    continue
                current += char
            if current.strip():
                params.append(current.strip())
            for param in params:
                # Parse: name: type = default
                # Handle cases like: client: [Client](xdk.md#xdk.Client)
                param_match = re.match(
                    r"(\w+)(?:\s*:\s*([^=]+))?(?:\s*=\s*(.+))?$", param.strip()
                )
                if param_match:
                    param_name = param_match.group(1)
                    param_type_raw = (
                        param_match.group(2).strip() if param_match.group(2) else "Any"
                    )
                    param_default = (
                        param_match.group(3).strip() if param_match.group(3) else None
                    )
                    # Clean type - handle markdown links first
                    # Pattern: [Type](path#anchor) -> extract just "Type"
                    link_match = re.search(r"\[([^\]]+)\]\(([^\)]+)\)", param_type_raw)
                    if link_match:
                        param_type = link_match.group(1)  # Use just the link text
                    else:
                        # No link, use the raw type but clean it
                        param_type = param_type_raw
                        # Remove file references and anchors
                        param_type = re.sub(r"[a-z_]+\.(md|py)#[^\s]*", "", param_type)
                        param_type = re.sub(r"#[^\s]*", "", param_type)
                        # Remove incomplete link patterns like "Client]("
                        param_type = re.sub(r"\]\([^\)]*$", "", param_type)
                        param_type = re.sub(r"\]\([^\)]*\)", "", param_type)
                    # Clean up the type string
                    param_type = param_type.replace("|", " or ").strip()
                    # Remove any trailing/leading brackets and parentheses
                    param_type = re.sub(r"^[\[\(]+", "", param_type)
                    param_type = re.sub(r"[\]\)]+$", "", param_type)
                    # Remove any remaining incomplete patterns
                    param_type = re.sub(r"\]\(.*$", "", param_type)
                    # Escape angle brackets for MDX
                    param_type = param_type.replace("<", "&lt;").replace(">", "&gt;")
                    # Clean up extra spaces
                    param_type = re.sub(r"\s+", " ", param_type).strip()
                    # If type is empty or just brackets, default to Any
                    if not param_type or param_type in ["[", "]", "()", "(", ")", "]("]:
                        param_type = "Any"
                    # Find param description
                    param_desc_match = re.search(
                        rf":param\s+{param_name}:\s*([^\n]+)", method_body
                    )
                    param_desc = (
                        param_desc_match.group(1).strip() if param_desc_match else ""
                    )
                    # Build ParamField - use path instead of name
                    # For Python methods, parameters are function arguments
                    # Use "path" location for most parameters, or "body" if it's clearly a request body
                    param_location = (
                        "body"
                        if param_name.lower() in ["body", "data", "payload", "request"]
                        else "path"
                    )
                    param_field = f'<ParamField path="{param_location}.{param_name}" type="{param_type}"'
                    if param_default:
                        # Escape default value
                        param_default_clean = (
                            param_default.replace('"', "&quot;")
                            .replace("<", "&lt;")
                            .replace(">", "&gt;")
                        )
                        param_field += f' default="{param_default_clean}"'
                    param_field += ">"
                    if param_desc:
                        param_field += f"\n{param_desc}\n"
                    param_field += "</ParamField>"
                    param_fields.append(param_field)
        # Build new method format
        new_method = f"### `{method_name}`\n\n"
        if description:
            new_method += f"{description}\n\n"
        if param_fields:
            new_method += "#### Parameters\n\n"
            new_method += "\n\n".join(param_fields) + "\n\n"
        if return_type:
            # Extract return description
            return_desc_match = re.search(r":param\s+Returns?:\s*([^\n]+)", method_body)
            return_desc = (
                return_desc_match.group(1).strip() if return_desc_match else ""
            )
            # Clean return type - handle markdown links
            return_type_clean = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", return_type)
            return_type_clean = re.sub(
                r"[a-z_]+\.(md|py)#[^\s]+", "", return_type_clean
            )
            return_type_clean = return_type_clean.replace("[", "").replace("]", "")
            return_type_clean = return_type_clean.replace("<", "&lt;").replace(
                ">", "&gt;"
            )
            return_type_clean = re.sub(
                r":param\s+\w+:\s*", "", return_type_clean
            ).strip()
            return_type_clean = re.sub(r"\s+", " ", return_type_clean).strip()
            new_method += "#### Returns\n\n"
            new_method += f"`{return_type_clean}`"
            if return_desc and return_desc != return_type_clean:
                new_method += f" - {return_desc}"
            new_method += "\n\n"
        return new_method

    # Convert method signatures to use ParamField components
    # Match: ### method_name(params) → ReturnType


    def process_all_methods(text):
        # Find all method definitions
        method_pattern = r"###\s+`?([^\n(]+)`?\s*\(([^)]*)\)(?:\s*→\s*([^\n]+))?"
        methods = list(re.finditer(method_pattern, text, re.MULTILINE))
        if not methods:
            return text
        # Build result by processing each method and replacing its entire section
        result_parts = []
        last_pos = 0
        for i, match in enumerate(methods):
            # Add content before this method
            result_parts.append(text[last_pos : match.start()])
            # Find where this method's body ends (next method or end of content)
            method_start = match.start()
            if i + 1 < len(methods):
                next_method_start = methods[i + 1].start()
            else:
                next_method_start = len(text)
            # Get the full method section to replace
            method_section = text[method_start:next_method_start]
            # Convert this method
            converted = convert_method(match)
            # Remove the old method content from the section
            result_parts.append(converted)
            last_pos = next_method_start
        # Add remaining content
        result_parts.append(text[last_pos:])
        return "".join(result_parts)

    content = process_all_methods(content)

    # Clean up any remaining :param lines that weren't converted
    content = re.sub(r":param\s+(\w+):\s*([^\n]+)", r"**`\1`** - \2", content)
    content = re.sub(r":param\s+Returns?:\s*([^\n]+)", r"**Returns:** \1", content)

    # Remove duplicate return descriptions
    content = re.sub(
        r"(#### Returns\n\n`[^\n]+`[^\n]+\n\n)(\*\*`[^\n]+\*\*[^\n]+\n)+",
        r"\1",
        content,
    )

    # Remove duplicate method descriptions
    content = re.sub(r"(### `[^\n]+`\n\n[^\n]+\n\n[^\n]+\n\n)(\1)", r"\1", content)

    # Clean up any remaining old format parameter lines after ParamField sections
    content = re.sub(r"(</ParamField>\n\n)(\*\*`[^\n]+\*\*[^\n]+\n)+", r"\1", content)

    # Remove duplicate "Returns:" text in return sections
    content = re.sub(
        r"(#### Returns\n\n`[^\n]+`)\s*-\s*\*\*`[^\n]+\*\*\s*-\s*([^\n]+)",
        r"\1 - \2",
        content,
    )

    # Remove duplicate class definitions that appear after the main class header
    # Pattern: ### `class xdk.module.ClassName` followed by description and parameters
    # Find the main class header (should be ## ClassName)
    main_class_match = re.search(
        r"##\s+([A-Z][a-zA-Z0-9_]+Client|Client|Paginator|OAuth2PKCEAuth|BaseModel)",
        content,
    )
    if main_class_match:
        # Everything after the main class header should not have duplicate class definitions
        before_main = content[: main_class_match.end()]
        after_main = content[main_class_match.end() :]

        # Remove any class definitions from after_main
        after_main = re.sub(
            r"###\s+`?class\s+xdk\.[^\n]+\n\n[^\n]+\n\n(?:####\s+Parameters[^\n]+\n\n<ParamField[^>]+>\s*</ParamField>\s*\n)?",
            "",
            after_main,
        )

        content = before_main + after_main

    # Remove duplicate parameter sections (#### Parameters appearing twice in a row)
    content = re.sub(
        r"(#### Parameters\n\n<ParamField[^>]+>\s*</ParamField>\s*\n)\1", r"\1", content
    )

    return content


def process_markdown_content(
    content: str, title: str, current_file_path: str, known_targets: Dict[str, str]
) -> str:
    """Process and clean markdown content for Mintlify."""
    # Remove Sphinx-specific elements
    content = re.sub(r"\[\[include:.*?\]\]", "", content)
    # Fix code block formatting
    content = re.sub(r"```python\n", "```python\n", content)
    # Remove Sphinx breadcrumbs
    content = re.sub(
        r"^\[[^\]]+\]\([^\)]+\)\s*/\s*.*\n?", "", content, flags=re.MULTILINE
    )
    # Remove auto-generated comments at the beginning
    # Pattern: "Auto-generated ..." followed by description paragraphs ending with "Generated automatically - do not edit manually."
    content = re.sub(
        r"^Auto-generated[^\n]+\n\n[^\n]+\n\n[^\n]+\n\nAll methods[^\n]+\n\nGenerated automatically[^\n]+\n\n",
        "",
        content,
        flags=re.MULTILINE,
    )
    # Also remove variations - single line or multiple paragraphs
    content = re.sub(r"^Auto-generated[^\n]+\n\n", "", content, flags=re.MULTILINE)
    content = re.sub(
        r"^This module provides[^\n]+\n\n", "", content, flags=re.MULTILINE
    )
    content = re.sub(r"^All methods[^\n]+\n\n", "", content, flags=re.MULTILINE)
    content = re.sub(
        r"^Generated automatically[^\n]+\n\n", "", content, flags=re.MULTILINE
    )
    # Fix method names first - remove escaped underscores (before other processing)
    content = fix_method_names(content)
    # Reorganize content into proper sections (Constructors, Methods, Properties)
    content = reorganize_class_structure(content, current_file_path)
    # Clean up class definitions - remove asterisks, format properly
    content = clean_class_definitions(content)
    # Improve method formatting - convert to better structure with ParamField
    content = improve_method_formatting(content)
    # Fix internal links to absolute Mintlify paths
    def fix_link(match):
        text = match.group(1)
        raw_link_path = match.group(2)
        hash_part = match.group(3) or ""
        # Skip absolute URLs
        if re.match(r"^(?:https?:|mailto:|tel:)", raw_link_path, re.I):
            return match.group(0)
        link_path = raw_link_path.replace(".md", "").replace(".rst", "")
        current_dir = str(Path(current_file_path).parent) if current_file_path else ""
        # Normalize path
        if current_dir:
            joined = Path(current_dir) / link_path
            target_path = str(joined).replace("\\", "/").replace("docs/", "")
        else:
            target_path = link_path.replace("\\", "/").replace("docs/", "")
        # Use known target if available
        base_name = Path(target_path).stem
        if base_name in known_targets and "/" not in target_path:
            target_path = f"{known_targets[base_name]}/{base_name}"
        return f"[{text}](/xdks/python/reference/{target_path}{hash_part})"
    content = re.sub(
        r"\[([^\]]+)\]\(([^)#]+?)(?:\.(?:md|rst))?(#[^)]+)?\)", fix_link, content
    )
    # Fix method signatures
    content = re.sub(r"### (.*?)\(", r"### `\1`(", content)
    # Add proper spacing
    content = re.sub(r"\n\n\n+", "\n\n", content)
    # Remove first H1 header (frontmatter title will be used)
    content = re.sub(r"^\s*#\s+[^\n]+\n+", "", content)
    # Escape generic type angle brackets (but preserve component tags)
    # Simple approach: escape angle brackets, then fix Badge tags
    content = re.sub(
        r"\b([A-Z][A-Za-z0-9_]*)<([^>\n]+)>",
        lambda m: f"{m.group(1)}&lt;{m.group(2).replace('<', '&lt;').replace('>', '&gt;')}&gt;",
        content,
    )
    # Fix any escaped Badge tags (they shouldn't have angle brackets anyway)
    content = content.replace("&lt;/Badge&gt;", "</Badge>")
    content = content.replace("Class&lt;/Badge&gt;", "Class</Badge>")
    content = content.replace("BaseModel&lt;/Badge&gt;", "BaseModel</Badge>")
    # Remove Table of Contents blocks
    content = re.sub(
        r"(^##\s+Table of contents\n[\s\S]*?)(?=^##\s+|^#\s+|\Z)",
        "",
        content,
        flags=re.MULTILINE | re.IGNORECASE,
    )
    # Fix asterisks around type annotations that break MDX parsing
    # Pattern: #### field *: Type* *= value* -> #### field : Type = value
    # This handles cases like: model_config *: ClassVar[ConfigDict]* *= {'extra': 'allow', ...}*
    # Match: field *: Type* *= value* (where value can contain quotes, commas, etc.)
    # Use a more specific pattern that includes the header marker
    content = re.sub(
        r"(####\s+\w+)\s+\*\s*:\s*([^*]+?)\s*\*\s*\*\s*=\s*([^\n]+?)\s*\*",
        r"\1: \2 = \3",
        content,
        flags=re.MULTILINE,
    )
    # Fix asterisks around type annotations without equals (just type)
    # Pattern: field *: Type* -> field : Type
    content = re.sub(
        r"(\w+)\s+\*\s*:\s*([^*\n]+?)\s*\*", r"\1: \2", content, flags=re.MULTILINE
    )
    # Fix remaining asterisks around equals signs (standalone)
    # Pattern: *= value* -> = value
    content = re.sub(r"\*\s*=\s*([^\n]+?)\s*\*", r"= \1", content, flags=re.MULTILINE)
    # Fix asterisks around class/function names in headers
    # Pattern: ### *class* name -> ### class name
    content = re.sub(r"###\s+\*\s*(\w+)\s*\*\s+", r"### \1 ", content)
    # Convert property headers with type annotations to Mintlify components
    # This must happen AFTER asterisk removal
    # Pattern: #### field: Type = value\n\nDescription
    # Convert to: <ResponseField name="field" type="Type">Description</ResponseField>
    def convert_property_header(match):
        field_name = match.group(1)
        type_and_default = match.group(2).strip()
        description = match.group(3).strip() if match.group(3) else ""
        # Split type and default value
        # Format: ClassVar[ConfigDict] = {'extra': 'allow', ...}
        type_match = re.match(r"([^=]+?)(?:\s*=\s*(.+))?$", type_and_default)
        if type_match:
            type_annotation = type_match.group(1).strip()
            default_value = type_match.group(2).strip() if type_match.group(2) else None
        else:
            type_annotation = type_and_default
            default_value = None
        # Clean up type annotation - remove ClassVar, brackets, etc.
        # ClassVar[ConfigDict] -> ConfigDict
        type_clean = re.sub(r"ClassVar\[([^\]]+)\]", r"\1", type_annotation)
        # Remove any remaining brackets for display
        type_clean = type_clean.replace("[", "").replace("]", "").strip()
        # Escape type for MDX
        type_clean = type_clean.replace("<", "&lt;").replace(">", "&gt;")
        # Build the component
        component = f'<ResponseField name="{field_name}" type="{type_clean}"'
        if default_value:
            # Escape default value for MDX - use code block for complex values
            if "{" in default_value or "[" in default_value:
                # For complex values, put in description instead
                description = f"Default: `{default_value}`\n\n{description}".strip()
            else:
                component += f' default="{default_value}"'
        component += ">"
        if description:
            component += f"\n{description}\n"
        component += "</ResponseField>"
        return component
    # Match: #### field: Type = value\n\nDescription
    # Match the entire header line, then blank line, then description
    content = re.sub(
        r"####\s+(\w+)\s*:\s*([^\n]+?)\s*\n\s*\n([^\n]+(?:\n(?!####)[^\n]+)*)?",
        convert_property_header,
        content,
        flags=re.MULTILINE,
    )
    return content


    def fix_link(match):
        text = match.group(1)
        raw_link_path = match.group(2)
        hash_part = match.group(3) or ""
        # Skip absolute URLs
        if re.match(r"^(?:https?:|mailto:|tel:)", raw_link_path, re.I):
            return match.group(0)
        link_path = raw_link_path.replace(".md", "").replace(".rst", "")
        current_dir = str(Path(current_file_path).parent) if current_file_path else ""
        # Normalize path
        if current_dir:
            joined = Path(current_dir) / link_path
            target_path = str(joined).replace("\\", "/").replace("docs/", "")
        else:
            target_path = link_path.replace("\\", "/").replace("docs/", "")
        # Use known target if available
        base_name = Path(target_path).stem
        if base_name in known_targets and "/" not in target_path:
            target_path = f"{known_targets[base_name]}/{base_name}"
        return f"[{text}](/xdks/python/reference/{target_path}{hash_part})"

    content = re.sub(
        r"\[([^\]]+)\]\(([^)#]+?)(?:\.(?:md|rst))?(#[^)]+)?\)", fix_link, content
    )

    # Fix method signatures
    content = re.sub(r"### (.*?)\(", r"### `\1`(", content)

    # Add proper spacing
    content = re.sub(r"\n\n\n+", "\n\n", content)

    # Remove first H1 header (frontmatter title will be used)
    content = re.sub(r"^\s*#\s+[^\n]+\n+", "", content)

    # Escape generic type angle brackets (but preserve component tags)
    # Simple approach: escape angle brackets, then fix Badge tags
    content = re.sub(
        r"\b([A-Z][A-Za-z0-9_]*)<([^>\n]+)>",
        lambda m: f"{m.group(1)}&lt;{m.group(2).replace('<', '&lt;').replace('>', '&gt;')}&gt;",
        content,
    )

    # Fix any escaped Badge tags (they shouldn't have angle brackets anyway)
    content = content.replace("&lt;/Badge&gt;", "</Badge>")
    content = content.replace("Class&lt;/Badge&gt;", "Class</Badge>")
    content = content.replace("BaseModel&lt;/Badge&gt;", "BaseModel</Badge>")

    # Remove Table of Contents blocks
    content = re.sub(
        r"(^##\s+Table of contents\n[\s\S]*?)(?=^##\s+|^#\s+|\Z)",
        "",
        content,
        flags=re.MULTILINE | re.IGNORECASE,
    )

    # Fix asterisks around type annotations that break MDX parsing
    # Pattern: #### field *: Type* *= value* -> #### field : Type = value
    # This handles cases like: model_config *: ClassVar[ConfigDict]* *= {'extra': 'allow', ...}*
    # Match: field *: Type* *= value* (where value can contain quotes, commas, etc.)
    # Use a more specific pattern that includes the header marker
    content = re.sub(
        r"(####\s+\w+)\s+\*\s*:\s*([^*]+?)\s*\*\s*\*\s*=\s*([^\n]+?)\s*\*",
        r"\1: \2 = \3",
        content,
        flags=re.MULTILINE,
    )

    # Fix asterisks around type annotations without equals (just type)
    # Pattern: field *: Type* -> field : Type
    content = re.sub(
        r"(\w+)\s+\*\s*:\s*([^*\n]+?)\s*\*", r"\1: \2", content, flags=re.MULTILINE
    )

    # Fix remaining asterisks around equals signs (standalone)
    # Pattern: *= value* -> = value
    content = re.sub(r"\*\s*=\s*([^\n]+?)\s*\*", r"= \1", content, flags=re.MULTILINE)

    # Fix asterisks around class/function names in headers
    # Pattern: ### *class* name -> ### class name
    content = re.sub(r"###\s+\*\s*(\w+)\s*\*\s+", r"### \1 ", content)

    # Convert property headers with type annotations to Mintlify components
    # This must happen AFTER asterisk removal
    # Pattern: #### field: Type = value\n\nDescription
    # Convert to: <ResponseField name="field" type="Type">Description</ResponseField>


    def convert_property_header(match):
        field_name = match.group(1)
        type_and_default = match.group(2).strip()
        description = match.group(3).strip() if match.group(3) else ""
        # Split type and default value
        # Format: ClassVar[ConfigDict] = {'extra': 'allow', ...}
        type_match = re.match(r"([^=]+?)(?:\s*=\s*(.+))?$", type_and_default)
        if type_match:
            type_annotation = type_match.group(1).strip()
            default_value = type_match.group(2).strip() if type_match.group(2) else None
        else:
            type_annotation = type_and_default
            default_value = None
        # Clean up type annotation - remove ClassVar, brackets, etc.
        # ClassVar[ConfigDict] -> ConfigDict
        type_clean = re.sub(r"ClassVar\[([^\]]+)\]", r"\1", type_annotation)
        # Remove any remaining brackets for display
        type_clean = type_clean.replace("[", "").replace("]", "").strip()
        # Escape type for MDX
        type_clean = type_clean.replace("<", "&lt;").replace(">", "&gt;")
        # Build the component
        component = f'<ResponseField name="{field_name}" type="{type_clean}"'
        if default_value:
            # Escape default value for MDX - use code block for complex values
            if "{" in default_value or "[" in default_value:
                # For complex values, put in description instead
                description = f"Default: `{default_value}`\n\n{description}".strip()
            else:
                component += f' default="{default_value}"'
        component += ">"
        if description:
            component += f"\n{description}\n"
        component += "</ResponseField>"
        return component

    # Match: #### field: Type = value\n\nDescription
    # Match the entire header line, then blank line, then description
    content = re.sub(
        r"####\s+(\w+)\s*:\s*([^\n]+?)\s*\n\s*\n([^\n]+(?:\n(?!####)[^\n]+)*)?",
        convert_property_header,
        content,
        flags=re.MULTILINE,
    )

    return content


def get_category_from_path(file_path: str) -> str:
    """Determine category from file path."""
    if "client" in file_path.lower() and "Client" in file_path:
        return "Getting Started"
    if "paginator" in file_path.lower():
        return "Core Features"
    if "stream" in file_path.lower():
        return "Core Features"
    return "API Reference"


def process_docs():
    """Main processing function."""
    try:
        # First, try to generate documentation
        print("📚 Generating documentation...")
        try:
            import subprocess
            result = subprocess.run(
                [sys.executable, "scripts/generate-docs-simple.py"],
                check=True,
                capture_output=True,
                text=True,
            )
            print(result.stdout)
        except subprocess.CalledProcessError as e:
            print("⚠️  Sphinx generation failed, using existing docs if available...")
            if not Path("docs").exists() or not any(Path("docs").iterdir()):
                raise RuntimeError(
                    "No documentation found and Sphinx generation failed. "
                    "Please install Sphinx: pip install sphinx myst-parser sphinx-markdown-builder"
                )
            print("✅ Using existing documentation files")
        # Create output directory
        output_dir = Path(MINTLIFY_CONFIG["outputDir"])
        if output_dir.exists():
            shutil.rmtree(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)
        # Create subdirectories
        (output_dir / "xdks" / "python" / "reference").mkdir(
            parents=True, exist_ok=True
        )
        print("📝 Processing markdown files...")
        # Get all markdown files
        docs_dir = Path("docs")
        files = list(docs_dir.rglob("*.md"))
        # Build map of known targets
        known_targets = {}
        for f in files:
            base = f.stem
            parent = f.parent.name
            if parent and parent != "docs" and base not in known_targets:
                known_targets[base] = parent
        processed_files = []
        navigation = {
            "Getting Started": [],
            "Core Features": [],
            "API Reference": [],
            "Authentication": [],
            "Utilities": [],
        }
        for file_path in files:
            if file_path.name == "README.md":
                continue
            # Special handling for modules.md - convert to accordion format
            if file_path.name == "modules.md":
                continue  # Will process separately
            content = file_path.read_text(encoding="utf-8")
            # Extract title
            title_match = re.search(r"^#\s+(.+)$", content, re.MULTILINE)
            title = title_match.group(1) if title_match else file_path.stem
            # Clean title using improved function (pass content for better extraction)
            cleaned_title = clean_title(title, str(file_path), content)
            category = get_category_from_path(str(file_path))
            processed_content = process_markdown_content(
                content,
                cleaned_title,
                str(file_path.relative_to(docs_dir)),
                known_targets,
            )
            # Generate frontmatter
            frontmatter = generate_frontmatter(
                cleaned_title, cleaned_title, str(file_path)
            )
            final_content = frontmatter + processed_content
            # Determine output path
            base_name = file_path.stem
            sub_dir = file_path.parent.name if file_path.parent != docs_dir else ""
            target_dir = output_dir / "xdks" / "python" / "reference" / sub_dir
            target_dir.mkdir(parents=True, exist_ok=True)
            output_path = target_dir / f"{base_name}.mdx"
            # Write processed file
            output_path.write_text(final_content, encoding="utf-8")
            processed_files.append(
                {
                    "title": cleaned_title,
                    "category": category,
                    "path": str(output_path),
                    "originalPath": str(file_path),
                }
            )
            # Add to navigation
            if category in navigation:
                relative_ref_path = f"{sub_dir}/{base_name}" if sub_dir else base_name
                navigation[category].append(
                    {
                        "title": cleaned_title,
                        "url": f"xdks/python/reference/{relative_ref_path}",
                    }
                )
        # Create high-level documentation pages
        print("📄 Creating high-level documentation pages...")
        # Overview page
        overview_content = """---
title: Python XDK
sidebarTitle: Overview
---
The Python XDK (X Developer Kit) is our official client library for interacting with the X API v2 using Python. It allows developers to get started with our API quickly and build applications with it. It is generated based on our official [OpenAPI specification](https://api.x.com/2/openapi.json). It abstracts away low-level HTTP details while providing fine-grained control when needed.
## Key Features
- 🔐 **OAuth Support**: Full support for Bearer Token (app-only) auth, OAuth 2.0 with PKCE (user context), and OAuth 1.0.
- 🔄 **Pagination**: Automatically page through large results. The XDK takes care of pagination without requiring you to make multiple API calls using the `next_token`.
- 📡 **Streaming**: Supports real-time data streaming for endpoints like filtered stream that require persistent http connection.
- 🎯 **Comprehensive Coverage**: Supports all X API v2 endpoints including such as search, timelines, filtered-stream and more.
**Version Compatibility**: Python 3.8+. Tested on CPython and PyPy.
**License**: [MIT License](https://github.com/xdevplatform/xdk/blob/main/LICENSE)
"""
        (output_dir / "xdks" / "python" / "overview.mdx").write_text(overview_content)
        # Install page
        install_content = """---
title: "Install"
sidebarTitle: "Install"
---
The XDK Python SDK is available directly from the GitHub repository and can be installed via `pip`.
## Prerequisites
- Python 3.8 or higher.
- `pip` and `venv` for virtual environments (recommended).
## Quick Install
Install the XDK from the GitHub subdirectory:
```bash
pip install xdk
```
This fetches the latest generated version from the `main` branch.
## Development Install
For development or contributing:
1. Clone the repository:
   ```bash
   git clone https://github.com/xdevplatform/xdk.git
   cd xdk/python
   ```
2. Install dependencies in editable mode:
   ```bash
   pip install -e .
   ```
   This installs the SDK and its runtime dependencies.
3. (Optional) Install dev dependencies for testing/linting:
   ```bash
   pip install -e .[dev]
   ```
## Verification
Test the installation:
```python
import xdk
print(xdk.__version__)  # Should print the XDK version
```
**Note:** Since the XDK is generated using the OpenAPI spec, always check the [X API changelog](https://docs.x.com/changelog) and XDK release notes in the repo for any changes.
"""
        (output_dir / "xdks" / "python" / "install.mdx").write_text(install_content)
        # Quickstart page
        quickstart_content = """---
title: Quickstart
sidebarTitle: Quickstart
---
This example showcases how to quickly search for Posts using the XDK using Bearer Token authentication.
## Step 1: Install the SDK
```bash
pip install xdk
```
## Step 2: Get Your Bearer Token
1. Log in to the [X Developer Portal](https://developer.x.com/en/portal/dashboard).
2. Create or select an app.
3. Under "Keys and Tokens," generate a Bearer Token (app-only auth).
## Step 3: Write and Run Your First Script
Create a file `quickstart.py`:
```python
# Import the client
from xdk import Client
# Replace with your actual Bearer Token
client = Client(bearer_token="YOUR_BEARER_TOKEN_HERE")
# Fetch recent Posts mentioning "api"
response = client.posts.search_recent(query="api", max_results=10)
# Print the first Post's text
if response.data:
    print(f"Latest Post: {response.data[0]['text']}")
else:
    print("No Posts found.")
```
Run it:
```bash
python quickstart.py
```
**Expected Output**:
```
Latest Post: Exciting updates on XDK Python SDK!
```
**Troubleshooting**: If you get a 401 error, double-check your Bearer Token. For rate limits (429), wait and retry.
## Next Steps
- Explore [Authentication](/xdks/python/authentication) to understand how to use Bearer Token (app-only) auth, OAuth 2.0 with PKCE (user context), and OAuth 1.0.
- Learn about [Pagination](/xdks/python/pagination) for use-cases where you want large number of results returned without worrying about making multiple API calls.
- Dive into [Streaming](/xdks/python/streaming) to learn how to work with real-time data.
"""
        (output_dir / "xdks" / "python" / "quickstart.mdx").write_text(
            quickstart_content
        )
        # Authentication page
        auth_content = """---
title: Authentication
sidebarTitle: Authentication
---
The X API requires authentication for all endpoints. The XDK supports three authentication methods:
1. Bearer Token (app-only)
2. OAuth 2.0 with PKCE
3. OAuth 1.0a User Context
- **Bearer Token**: Use this for read-only access for endpoints that support app-auth (e.g., searching Post's, streaming endpoints).
- **OAuth 2.0 PKCE**: Secure authentication for scope-based, user-authorized access (e.g. getting authenticated user's Post non_public metrics)
- **OAuth 1.0a**: Legacy auth for full read/write access, including DMs and media uploads.
**Note**: We recommend developers move away from OAuth 1.0 and use OAuth 2.0 for user-authorized access.
Obtain credentials from the [X Developer Portal](https://developer.x.com/en/portal/dashboard). You'll need an approved developer account and an app with appropriate permissions (e.g., Read + Write).
## Creating a Client
All authentication flows create a `Client` instance:
```python
from xdk import Client
```
### 1. Bearer Token (App-Only)
For read-only operations without user context.
**Steps**:
1. In the Developer Portal, generate a Bearer Token for your app.
2. Pass it to the `Client`.
**Example**:
```python
client = Client(bearer_token="XXXXX")
```
**Usage**:
```python
response = client.posts.search_recent(query="python", max_results=10)
print(response.data[0]['text'])  # Access first Post
```
### 2. OAuth 2.0 with PKCE (User Context)
This example shows how to use OAuth 2.0 with Proof Key for Code Exchange (PKCE). Use this for user-specific access (e.g. posting on behalf of a user), uploading media for a user etc.).
**Steps**:
1. In the developer portal, register your app with a redirect URI (e.g., `http://localhost:8080/callback`).
2. Get Client ID (no secret needed for PKCE).
3. Initiate the flow, direct user to auth URL and handle callback.
**Example** (using a web server for callback):
```python
from xdk.auth import OAuth2PKCE
from urllib.parse import urlparse
import webbrowser
# Step 1: Create PKCE instance
auth = OAuth2PKCE(
    client_id="your_client_id",
    redirect_uri="http://localhost:8080/callback",
    scopes=["tweet.read", "users.read", "offline.access"]  # Adjust scopes as needed
)
# Step 2: Get authorization URL
auth_url = auth.get_authorization_url()
print(f"Visit this URL to authorize: {auth_url}")
webbrowser.open(auth_url)
# Step 3: Handle callback (in a real app, use a web framework like Flask)
# Assume callback_url = "http://localhost:8080/callback?code=AUTH_CODE_HERE"
callback_url = input("Paste the full callback URL here: ")
parsed = urlparse(callback_url)
code = parsed.query.split("=")[1]
# Step 4: Exchange code for tokens
tokens = auth.fetch_token(authorization_code=code)
access_token = tokens["access_token"]
refresh_token = tokens["refresh_token"]  # Store for renewal
# Step 5: Create client
client = Client(oauth2_access_token=access_token)
```
**Token Refresh** (automatic in SDK for long-lived sessions):
```python
# If access token expires, refresh using stored refresh_token
tokens = auth.refresh_token(refresh_token=refresh_token)
client = Client(oauth2_access_token=tokens["access_token"])
```
### 3. OAuth 1.0a User Context
For legacy endpoints that require OAuth 1.0 support.
**Steps**:
1. Generate Consumer Key/Secret and Access Token/Secret via Developer Portal.
2. Pass it when initializing the client.
**Example**:
```python
from xdk.auth import OAuth1User
auth = OAuth1User(
    consumer_key="your_consumer_key",
    consumer_secret="your_consumer_secret",
    access_token="your_access_token",
    access_token_secret="your_access_token_secret"
)
client = Client(auth=auth)
```
**Note**:
- Never hardcode secrets in production; use environment variables or secret managers (e.g., `os.getenv("X_BEARER_TOKEN")`).
- For PKCE, ensure HTTPS for redirect URIs in production.
- The SDK validates tokens and raises `xdk.AuthenticationError` on failures.
"""
        (output_dir / "xdks" / "python" / "authentication.mdx").write_text(auth_content)
        # Pagination page
        pagination_content = """---
title: Pagination
sidebarTitle: Pagination
---
The X API uses pagination for endpoints that return multiple pages of results (e.g. timelines, search etc.). Each API call response includes a `meta` object with `result_count`, `previous_token`, and `next_token`. The XDK takes care of making multiple API calls using the `next_token` so developers can just specify how much data they are looking for without having to make multiple calls.
The SDK simplifies this with:
- **Built-in Iterators**: Use generator functions for seamless multi-page fetching.
- **Explicit Token Handling**: For flexible manual control when needed by passing `pagination_token` when needed.
- **Max Results Enforcement**: Respect `max_results` per call (up to API limits, e.g., 100 for search).
## Automatic Pagination (Recommended)
Use the `iterate()` method on paginated responses to fetch all results lazily.
**Example: Paginated Search**
```python
from xdk import Client
client = Client(bearer_token="your_bearer_token")
# Search with automatic pagination
all_posts = []
for page in client.posts.search_recent(
    query="python",
    max_results=100,  # Per page
    tweetfields=["created_at", "author_id"]  # Optional expansions
):
    all_posts.extend(page.data)
    print(f"Fetched {len(page.data)} Posts (total: {len(all_posts)})")
print(f"Total tweets: {len(all_posts)}")
```
- The iterator handles `next_token` automatically.
- Stops when no `next_token` is present.
- Supports rate limit backoff to avoid 429 errors.
## Manual Pagination
If you require control over the results for some custom logic (e.g. processing page-by-page), you can still use the `next_token` and do the pagination manually as shown below:
```python
response = client.posts.search_recent(
    query="xdk python sdk",
    max_results=100,
    pagination_token=None  # First page
)
print(f"First page: {len(response.data)} Posts")
next_token = response.meta.next_token
if next_token:
    next_response = client.posts.search_recent(
        query="xdk python sdk",
        max_results=100,
        pagination_token=next_token
    )
    print(f"Second page: {len(next_response.data)} Posts")
```
**Tips**:
- Always specify `max_results` to optimize (default varies by endpoint).
- Monitor `meta.result_count` for debugging.
- For very large queries, consider async iteration to avoid blocking.
"""
        (output_dir / "xdks" / "python" / "pagination.mdx").write_text(
            pagination_content
        )
        # Streaming page
        streaming_content = """---
title: Streaming
sidebarTitle: Streaming
---
The X API supports real-time data via endpoints like the [Filtered Stream Endpoint](https://docs.x.com/x-api/posts/filtered-stream/introduction), delivering matching Posts as they occur. This requires making a persistent http connection.
## Setup and Basic Streaming
### Synchronous
```python
from xdk import Client
# Initialize client
client = Client(bearer_token="your_bearer_token")
# Stream posts (make sure you have rules set up first)
for post_response in client.stream.posts():
    data = post_response.model_dump()
    if 'data' in data and data['data']:
        tweet = data['data']
        print(f"Post: {tweet.get('text', '')}")
```
### Async
```python
import asyncio
from asyncio import Queue
import threading
from xdk import Client
async def stream_posts_async(client: Client):
    queue = Queue()
    loop = asyncio.get_event_loop()
    stop = threading.Event()
    def run_stream():
        for post in client.stream.posts():
            if stop.is_set():
                break
            asyncio.run_coroutine_threadsafe(queue.put(post), loop)
        asyncio.run_coroutine_threadsafe(queue.put(None), loop)
    threading.Thread(target=run_stream, daemon=True).start()
    while True:
        post = await queue.get()
        if post is None:
            break
        data = post.model_dump()
        if 'data' in data and data['data']:
            print(f"Post: {data['data'].get('text', '')}")
    stop.set()
async def main():
    client = Client(bearer_token="your_bearer_token")
    await stream_posts_async(client)
asyncio.run(main())
```
## Rule Management
Rules define filters on what specific data you are looking for(e.g. keywords, users etc). You can learn more about how to build rules using [this guide](https://docs.x.com/x-api/posts/filtered-stream/integrate/build-a-rule)
**Adding Rules**:
```python
from xdk.stream.models import UpdateRulesRequest
# Add a rule
add_rules = {
    "add": [
        {"value": "from:xdevelopers", "tag": "official_updates"}
    ]
}
request_body = UpdateRulesRequest(**add_rules)
response = client.stream.update_rules(body=request_body)
```
**Deleting Rules**:
```python
from xdk.stream.models import UpdateRulesRequest
delete_rules = {
    "delete": {
        "ids": ["rule_id_1", "rule_id_2"]
    }
}
request_body = UpdateRulesRequest(**delete_rules)
response = client.stream.update_rules(body=request_body)
```
**Listing Rules**:
```python
response = client.stream.get_rules()
# Print rules
for rule in response.data:
    print(f"ID: {rule.id}, Value: {rule.value}, Tag: {rule.tag}")
```
For full rule syntax, see [X Streaming Rules Docs](https://developer.x.com/en/docs/twitter-api/tweets/filtered-stream/integrate/build-a-rule).
## Troubleshooting
- **403 Forbidden**: Invalid auth or insufficient permissions.
- **420 Enhance Your Calm**: Rate limited; wait and retry.
- **No Data**: Check rules with `get_rules()`; ensure matching Posts exist.
For more examples and API reference, see the inline docstrings (e.g., `help(client.tweets.search_recent)`) or the generated stubs in the source. Contribute feedback via the [GitHub repo](https://github.com/xdevplatform/xdk/tree/main/xdk/python).
"""
        (output_dir / "xdks" / "python" / "streaming.mdx").write_text(streaming_content)
        # Create navigation structure
        print("⚙️ Creating navigation structure...")
        # Build API Reference groups
        ref_root = output_dir / "xdks" / "python" / "reference"
        classes_dir = ref_root / "classes"
        modules_dir = ref_root / "modules"
        def list_files_no_ext(directory):
            try:
                return [
                    f"xdks/python/reference/{directory.name}/{f.stem}"
                    for f in directory.glob("*.mdx")
                ]
            except:
                return []
        classes_pages = list_files_no_ext(classes_dir) if classes_dir.exists() else []
        modules_pages = list_files_no_ext(modules_dir) if modules_dir.exists() else []
        # Group pages by module prefix
        MODULE_PREFIXES = [
            "AccountActivity",
            "Activity",
            "Communities",
            "CommunityNotes",
            "Compliance",
            "Connections",
            "DirectMessages",
            "General",
            "Lists",
            "Media",
            "Posts",
            "Spaces",
            "Stream",
            "Trends",
            "Usage",
            "Users",
            "Webhooks",
            "Client",
            "Paginator",
            "OAuth2",
        ]
        def group_pages(pages, kind):
            buckets = {}
            for p in pages:
                name = Path(p).stem
                group = None
                for pref in MODULE_PREFIXES:
                    if name.startswith(pref):
                        group = pref
                        break
                if not group:
                    if kind == "classes":
                        if name.endswith("Client"):
                            group = "Clients"
                        elif "Stream" in name:
                            group = "Streaming"
                        elif "Paginator" in name:
                            group = "Pagination"
                        else:
                            group = "Core"
                    else:
                        group = "Misc"
                if group not in buckets:
                    buckets[group] = []
                buckets[group].append(p)
            # Sort pages within groups
            for group in buckets:
                buckets[group].sort()
            return [{"group": k, "pages": v} for k, v in sorted(buckets.items())]
        class_groups = group_pages(classes_pages, "classes")
        module_groups = group_pages(modules_pages, "modules")
        # Process modules.md to create accordion structure
        modules_md_path = docs_dir / "modules.md"
        if modules_md_path.exists():
            modules_raw = modules_md_path.read_text(encoding="utf-8")
            # Extract title
            title_match = re.search(r"^#\s+(.+)$", modules_raw, re.MULTILINE)
            modules_title = title_match.group(1).strip() if title_match else "Modules"
            # Parse the modules structure and convert to accordion
            # Instead of parsing modules.md, use the actual reference files we have
            # Group by package name from actual files
            packages = {}
            # Get all reference files
            ref_root = output_dir / "xdks" / "python" / "reference"
            all_ref_files = [
                f for f in ref_root.glob("*.mdx") if f.stem not in ["modules", "index"]
            ]
            # Group files by package
            for ref_file in all_ref_files:
                name = ref_file.stem
                # Extract package name
                if ".client" in name:
                    package_name = name.replace("xdk.", "").replace(".client", "")
                    module_type = "Client"
                elif ".models" in name:
                    package_name = name.replace("xdk.", "").replace(".models", "")
                    module_type = "Models"
                else:
                    # Other modules like xdk.client, xdk.paginator
                    parts = name.replace("xdk.", "").split(".")
                    package_name = parts[0] if parts else "Core"
                    module_type = parts[-1].capitalize() if len(parts) > 1 else "Core"
                # Convert to display name
                package_display = " ".join(
                    word.capitalize() for word in package_name.split("_")
                )
                if package_display not in packages:
                    packages[package_display] = []
                # Add module link
                packages[package_display].append(
                    {"name": module_type, "link": f"xdks/python/reference/{name}"}
                )
            # Build accordion content
            package_accordions = []
            for package_name, modules in sorted(packages.items()):
                if modules:
                    module_items = "\n".join(
                        [
                            f'  - [{m["name"]}](/{m["link"]})'
                            for m in sorted(modules, key=lambda x: x["name"])
                        ]
                    )
                    package_accordions.append(
                        f'  <Accordion title="{package_name}">\n{module_items}\n  </Accordion>'
                    )
            # Build final modules content
            if package_accordions:
                modules_content = f"""---
title: "{modules_title}"
sidebarTitle: "{modules_title}"
---
<AccordionGroup>
<Accordion title="Packages">
{chr(10).join(package_accordions)}
</Accordion>
</AccordionGroup>
"""
            else:
                modules_content = f"""---
title: "{modules_title}"
sidebarTitle: "{modules_title}"
---
<AccordionGroup>
<Accordion title="Packages">
</Accordion>
</AccordionGroup>
"""
            # Write modules.mdx
            (output_dir / "xdks" / "python" / "reference" / "modules.mdx").write_text(
                modules_content, encoding="utf-8"
            )
        # Get all reference files for navigation
        ref_root = output_dir / "xdks" / "python" / "reference"
        all_ref_files = [
            f
            for f in ref_root.glob("*.mdx")
            if f.stem != "modules" and f.stem != "index"
        ]
        # Separate into clients, models, and other
        client_files = [f for f in all_ref_files if f.stem.endswith(".client")]
        model_files = [f for f in all_ref_files if f.stem.endswith(".models")]
        other_files = [
            f
            for f in all_ref_files
            if not f.stem.endswith(".client") and not f.stem.endswith(".models")
        ]
        # Group by package
        def group_by_package(files):
            buckets = {}
            for f in files:
                name = f.stem
                if ".client" in name:
                    package = name.replace("xdk.", "").replace(".client", "")
                elif ".models" in name:
                    package = name.replace("xdk.", "").replace(".models", "")
                else:
                    parts = name.replace("xdk.", "").split(".")
                    package = parts[0] if parts else "Core"
                # Convert to display name
                package_display = " ".join(
                    word.capitalize() for word in package.split("_")
                )
                if package_display not in buckets:
                    buckets[package_display] = []
                buckets[package_display].append(f"xdks/python/reference/{f.stem}")
            # Sort pages within groups
            for group in buckets:
                buckets[group].sort()
            return [{"group": k, "pages": v} for k, v in sorted(buckets.items())]
        client_groups = group_by_package(client_files)
        model_groups = group_by_package(model_files)
        other_groups = group_by_package(other_files)
        # Generate navigation JSON
        # Build navigation structure with packages and modules in sidebar
        api_ref_pages = ["xdks/python/reference/modules"]
        # Add client groups
        if client_groups:
            api_ref_pages.append({"group": "Clients", "pages": client_groups})
        # Add model groups
        if model_groups:
            api_ref_pages.append({"group": "Models", "pages": model_groups})
        # Add other groups
        if other_groups:
            api_ref_pages.append({"group": "Core", "pages": other_groups})
        python_sdk_navigation = {
            "tab": "Python SDK",
            "hidden": True,
            "pages": [
                "xdks/python/overview",
                "xdks/python/install",
                "xdks/python/quickstart",
                "xdks/python/authentication",
                "xdks/python/pagination",
                "xdks/python/streaming",
                {"group": "API Reference", "pages": api_ref_pages},
            ],
        }
        # Write navigation JSON
        nav_json_path = output_dir / "python-sdk-navigation.json"
        nav_json_path.write_text(json.dumps(python_sdk_navigation, indent=2))
        print("✅ Python SDK documentation processed successfully!")
        print(f"📁 Output directory: {output_dir}/")
        print(f"📊 Processed {len(processed_files)} files")
        print("\n🚀 Integration steps:")
        print("1. Copy the 'xdks/' folder to your existing Mintlify site")
        print(
            "2. Add the navigation structure from 'python-sdk-navigation.json' to your mintlify.json"
        )
        print("3. Push to your main branch to deploy")
    except Exception as error:
        print(f"❌ Error processing documentation: {error}")
        import traceback
        traceback.print_exc()
        sys.exit(1)


        def list_files_no_ext(directory):
            try:
                return [
                    f"xdks/python/reference/{directory.name}/{f.stem}"
                    for f in directory.glob("*.mdx")
                ]
            except:
                return []

        classes_pages = list_files_no_ext(classes_dir) if classes_dir.exists() else []
        modules_pages = list_files_no_ext(modules_dir) if modules_dir.exists() else []

        # Group pages by module prefix
        MODULE_PREFIXES = [
            "AccountActivity",
            "Activity",
            "Communities",
            "CommunityNotes",
            "Compliance",
            "Connections",
            "DirectMessages",
            "General",
            "Lists",
            "Media",
            "Posts",
            "Spaces",
            "Stream",
            "Trends",
            "Usage",
            "Users",
            "Webhooks",
            "Client",
            "Paginator",
            "OAuth2",
        ]


        def group_pages(pages, kind):
            buckets = {}
            for p in pages:
                name = Path(p).stem
                group = None
                for pref in MODULE_PREFIXES:
                    if name.startswith(pref):
                        group = pref
                        break
                if not group:
                    if kind == "classes":
                        if name.endswith("Client"):
                            group = "Clients"
                        elif "Stream" in name:
                            group = "Streaming"
                        elif "Paginator" in name:
                            group = "Pagination"
                        else:
                            group = "Core"
                    else:
                        group = "Misc"
                if group not in buckets:
                    buckets[group] = []
                buckets[group].append(p)
            # Sort pages within groups
            for group in buckets:
                buckets[group].sort()
            return [{"group": k, "pages": v} for k, v in sorted(buckets.items())]

        class_groups = group_pages(classes_pages, "classes")
        module_groups = group_pages(modules_pages, "modules")

        # Process modules.md to create accordion structure
        modules_md_path = docs_dir / "modules.md"
        if modules_md_path.exists():
            modules_raw = modules_md_path.read_text(encoding="utf-8")
            # Extract title
            title_match = re.search(r"^#\s+(.+)$", modules_raw, re.MULTILINE)
            modules_title = title_match.group(1).strip() if title_match else "Modules"

            # Parse the modules structure and convert to accordion
            # Instead of parsing modules.md, use the actual reference files we have
            # Group by package name from actual files
            packages = {}

            # Get all reference files
            ref_root = output_dir / "xdks" / "python" / "reference"
            all_ref_files = [
                f for f in ref_root.glob("*.mdx") if f.stem not in ["modules", "index"]
            ]

            # Group files by package
            for ref_file in all_ref_files:
                name = ref_file.stem
                # Extract package name
                if ".client" in name:
                    package_name = name.replace("xdk.", "").replace(".client", "")
                    module_type = "Client"
                elif ".models" in name:
                    package_name = name.replace("xdk.", "").replace(".models", "")
                    module_type = "Models"
                else:
                    # Other modules like xdk.client, xdk.paginator
                    parts = name.replace("xdk.", "").split(".")
                    package_name = parts[0] if parts else "Core"
                    module_type = parts[-1].capitalize() if len(parts) > 1 else "Core"

                # Convert to display name
                package_display = " ".join(
                    word.capitalize() for word in package_name.split("_")
                )

                if package_display not in packages:
                    packages[package_display] = []

                # Add module link
                packages[package_display].append(
                    {"name": module_type, "link": f"xdks/python/reference/{name}"}
                )

            # Build accordion content
            package_accordions = []
            for package_name, modules in sorted(packages.items()):
                if modules:
                    module_items = "\n".join(
                        [
                            f'  - [{m["name"]}](/{m["link"]})'
                            for m in sorted(modules, key=lambda x: x["name"])
                        ]
                    )
                    package_accordions.append(
                        f'  <Accordion title="{package_name}">\n{module_items}\n  </Accordion>'
                    )

            # Build final modules content
            if package_accordions:
                modules_content = f"""---
title: "{modules_title}"
sidebarTitle: "{modules_title}"
---

<AccordionGroup>

<Accordion title="Packages">

{chr(10).join(package_accordions)}

</Accordion>

</AccordionGroup>
"""
            else:
                modules_content = f"""---
title: "{modules_title}"
sidebarTitle: "{modules_title}"
---

<AccordionGroup>

<Accordion title="Packages">
</Accordion>

</AccordionGroup>
"""

            # Write modules.mdx
            (output_dir / "xdks" / "python" / "reference" / "modules.mdx").write_text(
                modules_content, encoding="utf-8"
            )

        # Get all reference files for navigation
        ref_root = output_dir / "xdks" / "python" / "reference"
        all_ref_files = [
            f
            for f in ref_root.glob("*.mdx")
            if f.stem != "modules" and f.stem != "index"
        ]

        # Separate into clients, models, and other
        client_files = [f for f in all_ref_files if f.stem.endswith(".client")]
        model_files = [f for f in all_ref_files if f.stem.endswith(".models")]
        other_files = [
            f
            for f in all_ref_files
            if not f.stem.endswith(".client") and not f.stem.endswith(".models")
        ]

        # Group by package


        def group_by_package(files):
            buckets = {}
            for f in files:
                name = f.stem
                if ".client" in name:
                    package = name.replace("xdk.", "").replace(".client", "")
                elif ".models" in name:
                    package = name.replace("xdk.", "").replace(".models", "")
                else:
                    parts = name.replace("xdk.", "").split(".")
                    package = parts[0] if parts else "Core"
                # Convert to display name
                package_display = " ".join(
                    word.capitalize() for word in package.split("_")
                )
                if package_display not in buckets:
                    buckets[package_display] = []
                buckets[package_display].append(f"xdks/python/reference/{f.stem}")
            # Sort pages within groups
            for group in buckets:
                buckets[group].sort()
            return [{"group": k, "pages": v} for k, v in sorted(buckets.items())]

        client_groups = group_by_package(client_files)
        model_groups = group_by_package(model_files)
        other_groups = group_by_package(other_files)

        # Generate navigation JSON
        # Build navigation structure with packages and modules in sidebar
        api_ref_pages = ["xdks/python/reference/modules"]

        # Add client groups
        if client_groups:
            api_ref_pages.append({"group": "Clients", "pages": client_groups})

        # Add model groups
        if model_groups:
            api_ref_pages.append({"group": "Models", "pages": model_groups})

        # Add other groups
        if other_groups:
            api_ref_pages.append({"group": "Core", "pages": other_groups})

        python_sdk_navigation = {
            "tab": "Python SDK",
            "hidden": True,
            "pages": [
                "xdks/python/overview",
                "xdks/python/install",
                "xdks/python/quickstart",
                "xdks/python/authentication",
                "xdks/python/pagination",
                "xdks/python/streaming",
                {"group": "API Reference", "pages": api_ref_pages},
            ],
        }

        # Write navigation JSON
        nav_json_path = output_dir / "python-sdk-navigation.json"
        nav_json_path.write_text(json.dumps(python_sdk_navigation, indent=2))

        print("✅ Python SDK documentation processed successfully!")
        print(f"📁 Output directory: {output_dir}/")
        print(f"📊 Processed {len(processed_files)} files")
        print("\n🚀 Integration steps:")
        print("1. Copy the 'xdks/' folder to your existing Mintlify site")
        print(
            "2. Add the navigation structure from 'python-sdk-navigation.json' to your mintlify.json"
        )
        print("3. Push to your main branch to deploy")

    except Exception as error:
        print(f"❌ Error processing documentation: {error}")
        import traceback

        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    process_docs()
