#!/usr/bin/env bash
###############################################################################
# test_nlp_comprehensive.sh - Comprehensive test of all NLP commands
#
# Tests ALL DSL patterns and detects bugs
###############################################################################

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPORT_FILE="${SCRIPT_DIR}/nlp_comprehensive_report.md"
FAIL_LOG="${SCRIPT_DIR}/nlp_failures.log"

# Colors
GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; CYAN='\033[0;36m'
BLUE='\033[0;34m'; BOLD='\033[1m'; DIM='\033[2m'; RESET='\033[0m'

# Counters
TOTAL=0; PASSED=0; FAILED=0; DSL_MATCH=0; LLM_FALLBACK=0

declare -a FAILURES=()
declare -a SLOW_COMMANDS=()

log_test() {
    local test_name="$1"
    local result="$2"
    local time_ms="$3"
    local method="$4"
    
    TOTAL=$((TOTAL + 1))
    if [[ "$result" == "PASS" ]]; then
        PASSED=$((PASSED + 1))
        echo -e "  ${GREEN}✓${RESET} ${test_name} ${DIM}(${time_ms}ms)${RESET} [${CYAN}${method}${RESET}]"
    else
        FAILED=$((FAILED + 1))
        echo -e "  ${RED}✗${RESET} ${test_name} ${DIM}(${time_ms}ms)${RESET} [${YELLOW}${method}${RESET}]"
        FAILURES+=("${test_name}: ${result}")
        echo "${test_name}: ${result}" >> "$FAIL_LOG"
    fi
    
    # Track slow commands (>500ms)
    if [[ $time_ms -gt 500 ]]; then
        SLOW_COMMANDS+=("${test_name}: ${time_ms}ms")
    fi
}

run_test() {
    local description="$1"
    local prompt="$2"
    local expected_contains="${3:-}"
    
    local start_time end_time duration_ms
    start_time=$(date +%s%N)
    
    local output method
    output=$(proxym chat "$prompt" --cli-mode 2>&1)
    
    end_time=$(date +%s%N)
    duration_ms=$(( (end_time - start_time) / 1000000 ))
    
    # Detect method
    if echo "$output" | grep -q "dsl_match"; then
        method="DSL"
        DSL_MATCH=$((DSL_MATCH + 1))
    elif echo "$output" | grep -qi "LLM.*unavailable\|502\|service.*error"; then
        method="LLM-ERROR"
        LLM_FALLBACK=$((LLM_FALLBACK + 1))
    else
        method="OTHER"
    fi
    
    # Check result
    if [[ -n "$expected_contains" ]]; then
        if echo "$output" | grep -q "$expected_contains"; then
            log_test "$description" "PASS" "$duration_ms" "$method"
        else
            log_test "$description" "Expected: $expected_contains" "$duration_ms" "$method"
        fi
    else
        # Just check if command was generated
        if echo "$output" | grep -q "proxym"; then
            log_test "$description" "PASS" "$duration_ms" "$method"
        else
            log_test "$description" "No command generated" "$duration_ms" "$method"
        fi
    fi
}

# ==== TESTS START HERE ====

echo -e "${BOLD}${CYAN}╔═══════════════════════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}${CYAN}║  Comprehensive NLP Test Suite                          ║${RESET}"
echo -e "${BOLD}${CYAN}║  Testing ALL DSL patterns & detecting bugs             ║${RESET}"
echo -e "${BOLD}${CYAN}╚═══════════════════════════════════════════════════════════╝${RESET}"
echo ""

# Clear previous failures
> "$FAIL_LOG"

# ── QUERY TESTS ───────────────────────────────────────────────────────────
echo -e "${BOLD}${BLUE}═══ Query Tests (DSL should match) ═══${RESET}"

run_test "status query" "show status" "GET /dashboard/system"
run_test "vms list" "show vms" "GET /dashboard/vms"
run_test "tickets list" "list tickets" "GET /dashboard/tickets"
run_test "board view" "kanban" "GET /dashboard/tickets/board/view"
run_test "sprints list" "list sprints" "GET /dashboard/sprints"
run_test "projects list" "list projects" "GET /dashboard/projects"
run_test "models list" "models" "GET /v1/models"
run_test "providers list" "providers" "GET /dashboard/providers"
run_test "logs view" "logs" "GET /dashboard/logs"
run_test "costs view" "show costs" "GET /dashboard/costs"
run_test "diagnose" "diagnose" "GET /tests/status"
run_test "tools list" "list tools" "GET /dashboard/tools"
run_test "jobs list" "jobs" "GET /dashboard/tools/jobs"
run_test "queue view" "queue" "GET /dashboard/tools/queue"

# ── TICKET COMMAND TESTS ────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ Ticket Command Tests ═══${RESET}"

run_test "create simple ticket" "create ticket for fix login bug" "ticket add"
run_test "create typed ticket" "create feature ticket for add search" "ticket add"
run_test "create priority ticket" "create high priority ticket for refactor" "ticket add"
run_test "create full ticket" "create high priority feature ticket for implement API" "ticket add"
run_test "create ticket in project" "create ticket in calculator for fix bug" "ticket add"
run_test "dispatch ticket" "dispatch TKT-12345" "dispatch"
run_test "assign ticket" "assign TKT-12345 to aider" "dispatch"
run_test "close ticket" "close TKT-12345" "close"
run_test "analyze ticket" "analyze TKT-12345" "analyze"

# Polish
run_test "utwórz ticket" "utwórz ticket dla napraw błąd" "ticket add"
run_test "utwórz w projekcie" "utwórz ticket w calculator dla fix" "ticket add"

# ── SPRINT COMMAND TESTS ────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ Sprint Command Tests ═══${RESET}"

run_test "create sprint" "create sprint MVP" "sprint create"
run_test "create sprint with goal" "create sprint Release with goal to launch product" "sprint create"
run_test "start sprint" "start sprint MVP" "sprint.*start"
run_test "close sprint" "close sprint MVP" "sprint.*close"
run_test "add ticket to sprint" "add TKT-12345 to sprint MVP" "sprint"

# Polish
run_test "utwórz sprint" "utwórz sprint Sprint1" "sprint create"

# ── PROJECT COMMAND TESTS ───────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ Project Command Tests ═══${RESET}"

run_test "register project" "register project myapp at /tmp/myapp" "project add"
run_test "add project with tool" "create project test using aider at /tmp/test" "project add"
run_test "remove project" "remove project oldproject" "project.*remove\|DELETE"

# Polish
run_test "zarejestruj projekt" "zarejestruj projekt mojapp w /tmp/mojapp" "project add"

# ── VM COMMAND TESTS ────────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ VM Command Tests ═══${RESET}"

run_test "create VM" "create vm aider for myproject" "POST /dashboard/vms"
run_test "start VM" "start vm-12345" "POST /dashboard/vms"
run_test "stop VM" "stop vm-12345" "POST /dashboard/vms"
run_test "restart VM" "restart vm-12345" "POST /dashboard/vms"
run_test "delete VM" "delete vm-12345" "DELETE /dashboard/vms"
run_test "snapshot VM" "snapshot vm-12345 as backup" "POST /dashboard/vms"
run_test "restore VM" "restore vm-12345 from backup" "POST /dashboard/vms"
run_test "switch project" "switch vm-12345 to otherproject" "POST /dashboard/vms"

# Polish
run_test "stwórz VM" "stwórz VM aider dla mojproj" "POST /dashboard/vms"

# ── BATCH & WORKFLOW TESTS ─────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ Batch & Workflow Tests ═══${RESET}"

run_test "dispatch all tickets" "dispatch all tickets for myproject" "batch"
run_test "run task" "run fix bug on myproject" "run"

# ── EDGE CASES & BUG DETECTION ───────────────────────────────────────────────
echo ""
echo -e "${BOLD}${BLUE}═══ Edge Cases & Bug Detection ═══${RESET}"

run_test "empty prompt" "" ""
run_test "only spaces" "   " ""
run_test "special chars" "create ticket for fix bug @#$%" "ticket add"
run_test "very long task" "create ticket for $(python3 -c 'print(\"a\"*500)')" "ticket add"
run_test "unicode text" "create ticket for 修复登录错误" "ticket add"
run_test "multiple spaces" "create    ticket    for    fix    bug" "ticket add"
run_test "mixed case" "CrEaTe TiCkEt FoR fIx BuG" "ticket add"

# ── GENERATE REPORT ─────────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}${CYAN}═══ Generating Report ═══${RESET}"

cat > "$REPORT_FILE" << EOF
# Comprehensive NLP Test Report

**Date:** $(date)
**Duration:** ~$(($TOTAL * 2)) seconds

## Summary

| Metric | Count |
|--------|-------|
| **Total Tests** | $TOTAL |
| **Passed** | $PASSED |
| **Failed** | $FAILED |
| **DSL Matches** | $DSL_MATCH |
| **LLM Errors** | $LLM_FALLBACK |
| **Pass Rate** | $(awk "BEGIN {printf \"%.1f%%\", ($PASSED/$TOTAL)*100}") |

## Coverage

### Queries Tested
- status, vms, tickets, board, sprints, projects
- models, providers, logs, costs, diagnose
- tools, jobs, queue

### Commands Tested
- Tickets: create (all variants), dispatch, assign, close, analyze
- Sprints: create, start, close, add ticket
- Projects: register, remove
- VMs: create, start, stop, restart, delete, snapshot, restore, switch
- Batch: dispatch all, run workflow

### Languages
- English (all commands)
- Polish (selected commands)

## Performance

### Slow Commands (>500ms)
EOF

if [[ ${#SLOW_COMMANDS[@]} -eq 0 ]]; then
    echo "None - all commands fast!" >> "$REPORT_FILE"
else
    for cmd in "${SLOW_COMMANDS[@]}"; do
        echo "- $cmd" >> "$REPORT_FILE"
    done
fi

cat >> "$REPORT_FILE" << EOF

## Failures

EOF

if [[ ${#FAILURES[@]} -eq 0 ]]; then
    echo "**No failures!** ✅" >> "$REPORT_FILE"
else
    echo "| Test | Error |" >> "$REPORT_FILE"
    echo "|------|-------|" >> "$REPORT_FILE"
    for failure in "${FAILURES[@]}"; do
        echo "| ${failure%%:*} | ${failure#*:} |" >> "$REPORT_FILE"
    done
fi

cat >> "$REPORT_FILE" << EOF

## DSL Schema Coverage

- **Version:** 2.0
- **Queries:** $(proxym chat "help" 2>&1 | grep -c "📖" || echo "N/A")
- **Commands:** $(proxym chat "help" 2>&1 | grep -c "⚡" || echo "N/A")

## Recommendations

EOF

if [[ $FAILED -eq 0 ]]; then
    echo "✅ All tests passing! NLP commands are fully functional." >> "$REPORT_FILE"
else
    echo "⚠️ **$FAILED tests failed** - review failure log at: \`$FAIL_LOG\`" >> "$REPORT_FILE"
fi

# ── FINAL SUMMARY ──────────────────────────────────────────────────────────
echo ""
echo -e "${BOLD}╔════════════════════════════════════════════════════════════╗${RESET}"
echo -e "${BOLD}║                    FINAL SUMMARY                           ║${RESET}"
echo -e "${BOLD}╠════════════════════════════════════════════════════════════╣${RESET}"
printf "${BOLD}║${RESET}  Total:     ${CYAN}%3d${RESET}  |  Passed: ${GREEN}%3d${RESET}  |  Failed: ${RED}%3d${RESET}         ║\n" $TOTAL $PASSED $FAILED
printf "${BOLD}║${RESET}  DSL:       ${CYAN}%3d${RESET}  |  LLM Error: ${YELLOW}%3d${RESET}                    ║\n" $DSL_MATCH $LLM_FALLBACK
printf "${BOLD}║${RESET}  Pass Rate: ${GREEN}%6s${RESET}                                   ║\n" "$(awk "BEGIN {printf \"%.1f%%\", ($PASSED/$TOTAL)*100}")"
echo -e "${BOLD}╠════════════════════════════════════════════════════════════╣${RESET}"
echo -e "${BOLD}║${RESET}  Report: ${REPORT_FILE}                      ${BOLD}║${RESET}"
echo -e "${BOLD}║${RESET}  Fail Log: ${FAIL_LOG}                      ${BOLD}║${RESET}"
echo -e "${BOLD}╚════════════════════════════════════════════════════════════╝${RESET}"

exit $(( FAILED > 0 ? 1 : 0 ))
