#!/usr/bin/env python3
"""
#exonware/xwquery/src/exonware/xwquery/parsers/sql_tokenizer.py
Production-grade SQL tokenizer and lexer.
Handles all SQL:2016 standard tokens with excellent error reporting.
Company: eXonware.com
Author: eXonware Backend Team
Email: connect@exonware.com
Version: 0.0.1.8
Generation Date: 28-Oct-2025
"""

from enum import Enum, auto
from typing import Optional
from dataclasses import dataclass
from ...errors import XWQueryParseError
# ==================== SQL Token Types ====================


class SQLTokenType(Enum):
    """SQL token types following SQL:2016 standard."""
    # Keywords (DML)
    SELECT = auto()
    FROM = auto()
    WHERE = auto()
    JOIN = auto()
    INNER = auto()
    LEFT = auto()
    RIGHT = auto()
    FULL = auto()
    OUTER = auto()
    CROSS = auto()
    ON = auto()
    USING = auto()
    GROUP = auto()
    BY = auto()
    HAVING = auto()
    ORDER = auto()
    LIMIT = auto()
    OFFSET = auto()
    INSERT = auto()
    INTO = auto()
    VALUES = auto()
    UPDATE = auto()
    SET = auto()
    DELETE = auto()
    # Keywords (DDL)
    CREATE = auto()
    ALTER = auto()
    DROP = auto()
    TABLE = auto()
    INDEX = auto()
    VIEW = auto()
    DATABASE = auto()
    SCHEMA = auto()
    # Keywords (Advanced)
    WITH = auto()
    RECURSIVE = auto()
    AS = auto()
    DISTINCT = auto()
    ALL = auto()
    UNION = auto()
    INTERSECT = auto()
    EXCEPT = auto()
    CASE = auto()
    WHEN = auto()
    THEN = auto()
    ELSE = auto()
    END = auto()
    EXISTS = auto()
    IN = auto()
    NOT = auto()
    # Data types
    INTEGER = auto()
    INT = auto()
    BIGINT = auto()
    SMALLINT = auto()
    DECIMAL = auto()
    NUMERIC = auto()
    REAL = auto()
    DOUBLE = auto()
    PRECISION = auto()
    FLOAT = auto()
    CHAR = auto()
    VARCHAR = auto()
    TEXT = auto()
    DATE = auto()
    TIME = auto()
    TIMESTAMP = auto()
    BOOLEAN = auto()
    # Logical operators
    AND = auto()
    OR = auto()
    # Special keywords
    NULL = auto()
    TRUE = auto()
    FALSE = auto()
    IS = auto()
    LIKE = auto()
    BETWEEN = auto()
    # Aggregate functions
    COUNT = auto()
    SUM = auto()
    AVG = auto()
    MIN = auto()
    MAX = auto()
    # Literals
    STRING_LITERAL = auto()
    NUMBER_LITERAL = auto()
    IDENTIFIER = auto()
    # Operators
    EQUALS = auto()           # =
    NOT_EQUALS = auto()       # != or <>
    LESS_THAN = auto()        # <
    GREATER_THAN = auto()     # >
    LESS_EQUALS = auto()      # <=
    GREATER_EQUALS = auto()   # >=
    PLUS = auto()             # +
    MINUS = auto()            # -
    MULTIPLY = auto()         # *
    DIVIDE = auto()           # /
    MODULO = auto()           # %
    CONCAT = auto()           # ||
    # Punctuation
    COMMA = auto()            # ,
    DOT = auto()              # .
    SEMICOLON = auto()        # ;
    LPAREN = auto()           # (
    RPAREN = auto()           # )
    LBRACKET = auto()         # [
    RBRACKET = auto()         # ]
    # Special
    STAR = auto()             # * (wildcard)
    COMMENT = auto()          # -- or /* */
    EOF = auto()              # End of file
    UNKNOWN = auto()          # Unknown token
# ==================== SQL Token ====================
@dataclass

class SQLToken:
    """
    Represents a single SQL token.
    Attributes:
        type: Token type
        value: Token value (original text)
        position: Character position in query
        line: Line number (1-indexed)
        column: Column number (1-indexed)
    """
    type: SQLTokenType
    value: str
    position: int
    line: int
    column: int

    def __repr__(self) -> str:
        return f"SQLToken({self.type.name}, '{self.value}', {self.line}:{self.column})"

    def __eq__(self, other) -> bool:
        if isinstance(other, SQLToken):
            return self.type == other.type and self.value == other.value
        return False
# ==================== SQL Keywords ====================
SQL_KEYWORDS = {
    # DML
    'SELECT': SQLTokenType.SELECT,
    'FROM': SQLTokenType.FROM,
    'WHERE': SQLTokenType.WHERE,
    'JOIN': SQLTokenType.JOIN,
    'INNER': SQLTokenType.INNER,
    'LEFT': SQLTokenType.LEFT,
    'RIGHT': SQLTokenType.RIGHT,
    'FULL': SQLTokenType.FULL,
    'OUTER': SQLTokenType.OUTER,
    'CROSS': SQLTokenType.CROSS,
    'ON': SQLTokenType.ON,
    'USING': SQLTokenType.USING,
    'GROUP': SQLTokenType.GROUP,
    'BY': SQLTokenType.BY,
    'HAVING': SQLTokenType.HAVING,
    'ORDER': SQLTokenType.ORDER,
    'LIMIT': SQLTokenType.LIMIT,
    'OFFSET': SQLTokenType.OFFSET,
    'INSERT': SQLTokenType.INSERT,
    'INTO': SQLTokenType.INTO,
    'VALUES': SQLTokenType.VALUES,
    'UPDATE': SQLTokenType.UPDATE,
    'SET': SQLTokenType.SET,
    'DELETE': SQLTokenType.DELETE,
    # DDL
    'CREATE': SQLTokenType.CREATE,
    'ALTER': SQLTokenType.ALTER,
    'DROP': SQLTokenType.DROP,
    'TABLE': SQLTokenType.TABLE,
    'INDEX': SQLTokenType.INDEX,
    'VIEW': SQLTokenType.VIEW,
    'DATABASE': SQLTokenType.DATABASE,
    'SCHEMA': SQLTokenType.SCHEMA,
    # Advanced
    'WITH': SQLTokenType.WITH,
    'RECURSIVE': SQLTokenType.RECURSIVE,
    'AS': SQLTokenType.AS,
    'DISTINCT': SQLTokenType.DISTINCT,
    'ALL': SQLTokenType.ALL,
    'UNION': SQLTokenType.UNION,
    'INTERSECT': SQLTokenType.INTERSECT,
    'EXCEPT': SQLTokenType.EXCEPT,
    'CASE': SQLTokenType.CASE,
    'WHEN': SQLTokenType.WHEN,
    'THEN': SQLTokenType.THEN,
    'ELSE': SQLTokenType.ELSE,
    'END': SQLTokenType.END,
    'EXISTS': SQLTokenType.EXISTS,
    'IN': SQLTokenType.IN,
    'NOT': SQLTokenType.NOT,
    # Data types
    'INTEGER': SQLTokenType.INTEGER,
    'INT': SQLTokenType.INT,
    'BIGINT': SQLTokenType.BIGINT,
    'SMALLINT': SQLTokenType.SMALLINT,
    'DECIMAL': SQLTokenType.DECIMAL,
    'NUMERIC': SQLTokenType.NUMERIC,
    'REAL': SQLTokenType.REAL,
    'DOUBLE': SQLTokenType.DOUBLE,
    'PRECISION': SQLTokenType.PRECISION,
    'FLOAT': SQLTokenType.FLOAT,
    'CHAR': SQLTokenType.CHAR,
    'VARCHAR': SQLTokenType.VARCHAR,
    'TEXT': SQLTokenType.TEXT,
    'DATE': SQLTokenType.DATE,
    'TIME': SQLTokenType.TIME,
    'TIMESTAMP': SQLTokenType.TIMESTAMP,
    'BOOLEAN': SQLTokenType.BOOLEAN,
    # Logical
    'AND': SQLTokenType.AND,
    'OR': SQLTokenType.OR,
    # Special
    'NULL': SQLTokenType.NULL,
    'TRUE': SQLTokenType.TRUE,
    'FALSE': SQLTokenType.FALSE,
    'IS': SQLTokenType.IS,
    'LIKE': SQLTokenType.LIKE,
    'BETWEEN': SQLTokenType.BETWEEN,
    # Aggregate functions
    'COUNT': SQLTokenType.COUNT,
    'SUM': SQLTokenType.SUM,
    'AVG': SQLTokenType.AVG,
    'MIN': SQLTokenType.MIN,
    'MAX': SQLTokenType.MAX,
}
# ==================== SQL Tokenizer ====================


class SQLTokenizer:
    """
    Production-grade SQL tokenizer.
    Features:
    - Complete SQL:2016 keyword support
    - String literals with escape sequences
    - Line and block comments
    - Quoted identifiers (double quotes, backticks, brackets)
    - Numeric literals (integers, floats, scientific notation)
    - Excellent error messages with context
    - Performance optimized
    Security:
    - Max query length validation
    - Max token count validation
    - Position tracking for error reporting
    """
    MAX_QUERY_LENGTH = 1_000_000  # 1MB
    MAX_TOKENS = 10_000

    def __init__(self, query: str):
        """
        Initialize SQL tokenizer.
        Args:
            query: SQL query string
        Raises:
            XWQueryParseError: If query exceeds length limit
        """
        if len(query) > self.MAX_QUERY_LENGTH:
            raise XWQueryParseError(
                f"Query too long: {len(query)} characters "
                f"(max {self.MAX_QUERY_LENGTH}). "
                f"This prevents DoS attacks."
            )
        self.query = query
        self.position = 0
        self.line = 1
        self.column = 1
        self.tokens: list[SQLToken] = []
    # ==================== Main Tokenization ====================

    def tokenize(self) -> list[SQLToken]:
        """
        Tokenize entire SQL query.
        Returns:
            List of SQL tokens
        Raises:
            XWQueryParseError: On tokenization errors
        """
        while self.position < len(self.query):
            # Skip whitespace
            if self._current_char().isspace():
                self._skip_whitespace()
                continue
            # Comments
            if self._is_line_comment():
                self._skip_line_comment()
                continue
            if self._is_block_comment():
                self._skip_block_comment()
                continue
            # String literals
            if self._current_char() == "'":
                self.tokens.append(self._read_string_literal())
                continue
            # Numbers
            if self._current_char().isdigit():
                self.tokens.append(self._read_number_literal())
                continue
            # Negative numbers
            if self._current_char() == '-' and self._peek_char().isdigit():
                self.tokens.append(self._read_number_literal())
                continue
            # Identifiers and keywords
            if self._current_char().isalpha() or self._current_char() == '_':
                self.tokens.append(self._read_identifier_or_keyword())
                continue
            # Quoted identifiers
            if self._current_char() in ('"', '`', '['):
                self.tokens.append(self._read_quoted_identifier())
                continue
            # Operators and punctuation
            token = self._read_operator_or_punctuation()
            if token:
                self.tokens.append(token)
                continue
            # Unknown character
            self._error(f"Unexpected character: '{self._current_char()}'")
        # Validate token count
        if len(self.tokens) > self.MAX_TOKENS:
            raise XWQueryParseError(
                f"Too many tokens: {len(self.tokens)} (max {self.MAX_TOKENS}). "
                f"This prevents DoS attacks."
            )
        # Add EOF token
        self.tokens.append(SQLToken(
            SQLTokenType.EOF,
            '',
            self.position,
            self.line,
            self.column
        ))
        return self.tokens
    # ==================== Character Navigation ====================

    def _current_char(self) -> str:
        """Get current character."""
        if self.position >= len(self.query):
            return ''
        return self.query[self.position]

    def _peek_char(self, offset: int = 1) -> str:
        """Peek ahead at character."""
        pos = self.position + offset
        if pos >= len(self.query):
            return ''
        return self.query[pos]

    def _advance(self) -> str:
        """Advance to next character."""
        if self.position < len(self.query):
            char = self.query[self.position]
            self.position += 1
            self.column += 1
            if char == '\n':
                self.line += 1
                self.column = 1
            return char
        return ''

    def _skip_whitespace(self) -> None:
        """Skip whitespace characters."""
        while self._current_char().isspace():
            self._advance()
    # ==================== Comments ====================

    def _is_line_comment(self) -> bool:
        """Check if current position is line comment start."""
        return (self._current_char() == '-' and 
                self._peek_char() == '-')

    def _is_block_comment(self) -> bool:
        """Check if current position is block comment start."""
        return (self._current_char() == '/' and 
                self._peek_char() == '*')

    def _skip_line_comment(self) -> None:
        """Skip line comment (-- to end of line)."""
        while self._current_char() and self._current_char() != '\n':
            self._advance()

    def _skip_block_comment(self) -> None:
        """Skip block comment (/* to */)."""
        self._advance()  # /
        self._advance()  # *
        while self._current_char():
            if self._current_char() == '*' and self._peek_char() == '/':
                self._advance()  # *
                self._advance()  # /
                return
            self._advance()
        self._error("Unterminated block comment")
    # ==================== String Literals ====================

    def _read_string_literal(self) -> SQLToken:
        """Read string literal (single-quoted)."""
        start_pos = self.position
        start_line = self.line
        start_col = self.column
        self._advance()  # Opening '
        value = ''
        while self._current_char() and self._current_char() != "'":
            if self._current_char() == '\\':
                # Escape sequence
                self._advance()
                escaped = self._current_char()
                if escaped == 'n':
                    value += '\n'
                elif escaped == 't':
                    value += '\t'
                elif escaped == 'r':
                    value += '\r'
                elif escaped == '\\':
                    value += '\\'
                elif escaped == "'":
                    value += "'"
                else:
                    value += escaped
                self._advance()
            elif self._current_char() == "'" and self._peek_char() == "'":
                # SQL escape: '' -> '
                value += "'"
                self._advance()
                self._advance()
            else:
                value += self._advance()
        if self._current_char() != "'":
            self._error("Unterminated string literal")
        self._advance()  # Closing '
        return SQLToken(
            SQLTokenType.STRING_LITERAL,
            value,
            start_pos,
            start_line,
            start_col
        )
    # ==================== Number Literals ====================

    def _read_number_literal(self) -> SQLToken:
        """Read number literal (integer or float)."""
        start_pos = self.position
        start_line = self.line
        start_col = self.column
        value = ''
        # Optional negative sign
        if self._current_char() == '-':
            value += self._advance()
        # Integer part
        while self._current_char().isdigit():
            value += self._advance()
        # Decimal part
        if self._current_char() == '.' and self._peek_char().isdigit():
            value += self._advance()  # .
            while self._current_char().isdigit():
                value += self._advance()
        # Scientific notation
        if self._current_char() in ('e', 'E'):
            value += self._advance()
            if self._current_char() in ('+', '-'):
                value += self._advance()
            while self._current_char().isdigit():
                value += self._advance()
        return SQLToken(
            SQLTokenType.NUMBER_LITERAL,
            value,
            start_pos,
            start_line,
            start_col
        )
    # ==================== Identifiers ====================

    def _read_identifier_or_keyword(self) -> SQLToken:
        """Read identifier or keyword."""
        start_pos = self.position
        start_line = self.line
        start_col = self.column
        value = ''
        while self._current_char() and (
            self._current_char().isalnum() or 
            self._current_char() in ('_', '$')
        ):
            value += self._advance()
        # Check if keyword
        upper_value = value.upper()
        if upper_value in SQL_KEYWORDS:
            token_type = SQL_KEYWORDS[upper_value]
        else:
            token_type = SQLTokenType.IDENTIFIER
        return SQLToken(
            token_type,
            value,
            start_pos,
            start_line,
            start_col
        )

    def _read_quoted_identifier(self) -> SQLToken:
        """Read quoted identifier (double quotes, backticks, or brackets)."""
        start_pos = self.position
        start_line = self.line
        start_col = self.column
        quote = self._current_char()
        close_quote = {'"': '"', '`': '`', '[': ']'}[quote]
        self._advance()  # Opening quote
        value = ''
        while self._current_char() and self._current_char() != close_quote:
            value += self._advance()
        if self._current_char() != close_quote:
            self._error(f"Unterminated quoted identifier (expected {close_quote})")
        self._advance()  # Closing quote
        return SQLToken(
            SQLTokenType.IDENTIFIER,
            value,
            start_pos,
            start_line,
            start_col
        )
    # ==================== Operators & Punctuation ====================

    def _read_operator_or_punctuation(self) -> Optional[SQLToken]:
        """Read operator or punctuation."""
        start_pos = self.position
        start_line = self.line
        start_col = self.column
        char = self._current_char()
        next_char = self._peek_char()
        # Two-character operators
        two_char = char + next_char
        if two_char == '!=':
            self._advance()
            self._advance()
            return SQLToken(SQLTokenType.NOT_EQUALS, two_char, start_pos, start_line, start_col)
        elif two_char == '<>':
            self._advance()
            self._advance()
            return SQLToken(SQLTokenType.NOT_EQUALS, two_char, start_pos, start_line, start_col)
        elif two_char == '<=':
            self._advance()
            self._advance()
            return SQLToken(SQLTokenType.LESS_EQUALS, two_char, start_pos, start_line, start_col)
        elif two_char == '>=':
            self._advance()
            self._advance()
            return SQLToken(SQLTokenType.GREATER_EQUALS, two_char, start_pos, start_line, start_col)
        elif two_char == '||':
            self._advance()
            self._advance()
            return SQLToken(SQLTokenType.CONCAT, two_char, start_pos, start_line, start_col)
        # Single-character tokens
        token_map = {
            '=': SQLTokenType.EQUALS,
            '<': SQLTokenType.LESS_THAN,
            '>': SQLTokenType.GREATER_THAN,
            '+': SQLTokenType.PLUS,
            '-': SQLTokenType.MINUS,
            '*': SQLTokenType.STAR,
            '/': SQLTokenType.DIVIDE,
            '%': SQLTokenType.MODULO,
            ',': SQLTokenType.COMMA,
            '.': SQLTokenType.DOT,
            ';': SQLTokenType.SEMICOLON,
            '(': SQLTokenType.LPAREN,
            ')': SQLTokenType.RPAREN,
            '[': SQLTokenType.LBRACKET,
            ']': SQLTokenType.RBRACKET,
        }
        if char in token_map:
            self._advance()
            return SQLToken(token_map[char], char, start_pos, start_line, start_col)
        return None
    # ==================== Error Handling ====================

    def _error(self, message: str) -> None:
        """
        Raise parse error with context.
        Args:
            message: Error message
        Raises:
            XWQueryParseError: Always
        """
        # Get context (5 chars before and after)
        start = max(0, self.position - 5)
        end = min(len(self.query), self.position + 5)
        context = self.query[start:end]
        # Mark error position in context
        marker_pos = self.position - start
        context_with_marker = (
            context[:marker_pos] + 
            '>>>' + 
            context[marker_pos:]
        )
        error_msg = (
            f"SQL Tokenization Error: {message}\n"
            f"  Location: Line {self.line}, Column {self.column}\n"
            f"  Position: Character {self.position}\n"
            f"  Context: {context_with_marker}"
        )
        raise XWQueryParseError(error_msg)
    # ==================== Public API ====================

    def get_tokens(self) -> list[SQLToken]:
        """Get tokenized SQL tokens (cached after first call)."""
        if not self.tokens or self.tokens[-1].type != SQLTokenType.EOF:
            return self.tokenize()
        return self.tokens
# ==================== Convenience Function ====================


def tokenize_sql(query: str) -> list[SQLToken]:
    """
    Tokenize SQL query.
    Args:
        query: SQL query string
    Returns:
        List of SQL tokens
    Raises:
        XWQueryParseError: On tokenization errors
    """
    tokenizer = SQLTokenizer(query)
    return tokenizer.tokenize()
__all__ = [
    'SQLTokenType',
    'SQLToken',
    'SQLTokenizer',
    'tokenize_sql'
]
