#!/usr/bin/env bash
###############################################################################
# test_nlp_commands.sh - Comprehensive test of NLP command availability
#
# Tests:
#   1. DSL pattern matching (no LLM required)
#   2. CLI schema generation
#   3. NLP command coverage
###############################################################################

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPORT_FILE="${SCRIPT_DIR}/nlp_test_report.md"

GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'
CYAN='\033[0;36m'; BOLD='\033[1m'; DIM='\033[2m'; RESET='\033[0m'

_ok()   { echo -e "  ${GREEN}✓${RESET} $1"; }
_warn() { echo -e "  ${YELLOW}⚠${RESET} $1"; }
_fail() { echo -e "  ${RED}✗${RESET} $1"; }
_info() { echo -e "  ${DIM}$1${RESET}"; }
_hdr()  { echo -e "\n${BOLD}${CYAN}═══ $1 ═══${RESET}"; }

# ── Test Results ────────────────────────────────────────────────────────────
declare -a DSL_PASSED=()
declare -a DSL_FAILED=()
declare -a NLP_PASSED=()
declare -a NLP_FAILED=()

# ── DSL Test Functions ──────────────────────────────────────────────────────

test_dsl_query() {
    local prompt="$1"
    local expected_pattern="${2:-}"

    echo -e "  ${CYAN}DSL:${RESET} \"$prompt\""

    local output
    output=$(proxym chat "$prompt" 2>&1)

    # Check if DSL pattern matched (no LLM error)
    if echo "$output" | grep -qE "(dsl_match|✓|✗)"; then
        _ok "DSL pattern matched"
        DSL_PASSED+=("$prompt")
        return 0
    elif echo "$output" | grep -qi "LLM"; then
        _warn "Fell back to LLM (DSL pattern missing)"
        DSL_FAILED+=("$prompt")
        return 1
    else
        _fail "Unknown response"
        DSL_FAILED+=("$prompt")
        return 1
    fi
}

# ── NLP CLI Mode Test ───────────────────────────────────────────────────────

test_nlp_cli() {
    local prompt="$1"
    local expected_cmd_fragment="${2:-}"

    echo -e "  ${CYAN}NLP CLI:${RESET} \"$prompt\""

    local output
    output=$(proxym chat "$prompt" --cli-mode 2>&1)

    if echo "$output" | grep -q "proxym"; then
        _ok "Generated CLI command"
        NLP_PASSED+=("$prompt")
        return 0
    elif echo "$output" | grep -qi "LLM.*error\|API key\|unavailable"; then
        _warn "LLM unavailable (expected in test)"
        NLP_FAILED+=("$prompt (LLM unavailable)")
        return 1
    else
        _fail "Could not generate command"
        NLP_FAILED+=("$prompt")
        return 1
    fi
}

# ── Schema Test ─────────────────────────────────────────────────────────────

test_schema() {
    _hdr "Testing CLI Schema Generation"

    echo ""
    _info "Generating schema via Python..."

    python3 << 'PYTHON'
import json
from proxym.cli._cli_schema import get_cli_schema, get_cli_schema_text, format_schema_for_llm

schema = get_cli_schema()
print(f"Schema version: {schema.get('schema_version', 'unknown')}")
print(f"CLI name: {schema.get('cli_name', 'unknown')}")
print(f"Commands: {len(schema.get('commands', []))}")
print(f"Usage patterns: {len(schema.get('usage_patterns', []))}")

# Check top-level commands
print("\n--- Top-Level Commands ---")
for cmd in schema.get('commands', [])[:10]:
    name = cmd.get('name', 'unknown')
    sub = cmd.get('subcommands')
    sub_count = len(sub) if sub else 0
    print(f"  {name} (sub: {sub_count})")

print("\n--- Sample Usage Patterns ---")
for p in schema.get('usage_patterns', [])[:15]:
    print(f"  {p['pattern']}")

# Test formatting
print("\n--- Schema Text Sample ---")
text = get_cli_schema_text()
print(text[:500] + "...")
PYTHON
}

# ── DSL Coverage Test ────────────────────────────────────────────────────────

test_dsl_coverage() {
    _hdr "Testing DSL Pattern Coverage"

    # Status queries
    test_dsl_query "show status"
    test_dsl_query "status"
    test_dsl_query "jak idzie"

    # List queries
    test_dsl_query "list tickets"
    test_dsl_query "pokaż tickety"
    test_dsl_query "tickets"

    test_dsl_query "list projects"
    test_dsl_query "projekty"
    test_dsl_query "projects"

    test_dsl_query "list vms"
    test_dsl_query "pokaż VM-y"
    test_dsl_query "vm list"

    # Board
    test_dsl_query "board"
    test_dsl_query "kanban"
    test_dsl_query "tablica"

    # Sprints
    test_dsl_query "list sprints"
    test_dsl_query "sprinty"
    test_dsl_query "show sprints"

    # Models
    test_dsl_query "models"
    test_dsl_query "jakie modele"

    # Logs
    test_dsl_query "logs"
    test_dsl_query "logi"
    test_dsl_query "show logs"

    # Costs
    test_dsl_query "costs"
    test_dsl_query "koszty"

    # Diagnose
    test_dsl_query "diagnose"
    test_dsl_query "sprawdź"

    # Providers
    test_dsl_query "providers"
    test_dsl_query "dostawcy"
}

# ── NLP Command Generation Test ─────────────────────────────────────────────

test_nlp_commands() {
    _hdr "Testing NLP Command Generation (--cli-mode)"

    # Project management
    test_nlp_cli "register project test at /tmp/test using aider"
    test_nlp_cli "create project myproject at /home/user/project"

    # Sprint management
    test_nlp_cli "create sprint MySprint with goal to build feature"
    test_nlp_cli "list all sprints"

    # Ticket management
    test_nlp_cli "create ticket to implement calculator"
    test_nlp_cli "create high priority feature ticket in calculator"
    test_nlp_cli "list all tickets for calculator project"

    # Dispatch
    test_nlp_cli "dispatch ticket TKT-001 using aider"
    test_nlp_cli "dispatch all tickets for calculator project"

    # Analysis
    test_nlp_cli "analyze ticket TKT-001"
    test_nlp_cli "show analysis for TKT-001"

    # Status
    test_nlp_cli "check system status"
    test_nlp_cli "show proxym status"
}

# ── Generate Report ────────────────────────────────────────────────────────

generate_report() {
    _hdr "Generating Report"

    cat > "$REPORT_FILE" << EOF
# NLP Commands Test Report

Generated: $(date)

## Summary

### DSL Pattern Tests
- **Passed**: ${#DSL_PASSED[@]}
- **Failed**: ${#DSL_FAILED[@]}

### NLP Command Generation Tests
- **Passed**: ${#NLP_PASSED[@]}
- **Failed**: ${#NLP_FAILED[@]}

## DSL Patterns Working (No LLM Required)

| Command | Status |
|---------|--------|
EOF

    for cmd in "${DSL_PASSED[@]}"; do
        echo "| \`$cmd\` | ✅ |" >> "$REPORT_FILE"
    done

    for cmd in "${DSL_FAILED[@]}"; do
        echo "| \`$cmd\` | ❌ |" >> "$REPORT_FILE"
    done

    cat >> "$REPORT_FILE" << EOF

## NLP Commands Tested

| Command | Status |
|---------|--------|
EOF

    for cmd in "${NLP_PASSED[@]}"; do
        echo "| \`$cmd\` | ✅ |" >> "$REPORT_FILE"
    done

    for cmd in "${NLP_FAILED[@]}"; do
        echo "| \`$cmd\` | ⚠️ |" >> "$REPORT_FILE"
    done

    cat >> "$REPORT_FILE" << EOF

## Available DSL Patterns (Bundled)

### Queries
- status, jak idzie, co działa, pokaż stan, show status
- pokaż VM-y, lista maszyn, vm list, show vms
- ile wydałem, koszty, costs, show costs, budget
- logi, błędy, logs, show logs, errors
- jakie modele, models, list models
- list tickets, show tickets, pokaż tickety, zadania
- board, kanban, tablica
- list sprints, sprinty, show sprints
- diagnose, diagnostics, run tests, sprawdź, testy
- list projects, projekty, show projects
- providers, list providers, providery, dostawcy

### Commands
- stwórz VM {tool} dla {project}, create vm {tool} for {project}
- uruchom {vm_id}, start {vm_id}
- zatrzymaj {vm_id}, stop {vm_id}
- snapshot {vm_id} jako {name}, snapshot {vm_id} as {name}
- przełącz {vm_id} na {project}, switch {vm_id} to {project}
- dodaj konto {provider} klucz {key}, add account {provider} key {key}

## Recommendations

1. **Extend DSL patterns** to cover common project/ticket management commands
2. **Add more Polish patterns** for better i18n support
3. **Test with LLM available** to verify full NLP pipeline
4. **Document fallback behavior** when DSL doesn't match

## CLI Schema Coverage

The generated schema includes:
- 38 top-level commands
- 38 usage patterns

See schema output above for full details.

---
*Report generated by test_nlp_commands.sh*
EOF

    _ok "Report saved to: $REPORT_FILE"
}

# ── Main ────────────────────────────────────────────────────────────────────

echo -e "${BOLD}${CYAN}"
echo "  ╔═══════════════════════════════════════════════════════════╗"
echo "  ║  NLP Commands Comprehensive Test                        ║"
echo "  ║  Testing DSL patterns & CLI schema coverage             ║"
echo "  ╚═══════════════════════════════════════════════════════════╝"
echo -e "${RESET}"

# Run tests
test_schema
test_dsl_coverage
test_nlp_commands
generate_report

# Final summary
echo ""
echo -e "${BOLD}╔════════════════════════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║                   Test Summary                              ║${RESET}"
echo -e "${BOLD}╠════════════════════════════════════════════════════════════╣${RESET}"
echo -e "${BOLD}║${RESET}  DSL Patterns: ${#DSL_PASSED[@]} passed, ${#DSL_FAILED[@]} fallback to LLM"
echo -e "${BOLD}║${RESET}  NLP Commands: ${#NLP_PASSED[@]} generated, ${#NLP_FAILED[@]} failed"
echo -e "${BOLD}║${RESET}  Report: ${REPORT_FILE}"
echo -e "${BOLD}╚════════════════════════════════════════════════════════════╝${RESET}"
echo ""
