#!/usr/bin/env python3
"""ACHEM - Deep Web Research Tool"""

import sys
import re
import argparse
import time
import asyncio
import os
from datetime import datetime
from pathlib import Path

from .wikipedia_client import WikipediaClient
from .duckduckgo_client import get_ddg_client
from .text_analyzer import TextAnalyzer
from .context_classifier import classify_and_filter
from .positional_ranker import PositionalRanker, rank_by_structure
from .huggingface_summarizer import get_hf_summarizer
from .groq_summarizer import get_groq_summarizer
from .gemini_summarizer import get_gemini_summarizer
from .openrouter_summarizer import get_openrouter_summarizer
from .local_summarizer import get_local_summarizer
from .sqlite_cache import get_sqlite_cache
from .config_manager import ConfigManager
from .content_processor import ContentProcessor, process_content
from .output_formatter import (
    console,
    detect_theme,
    ThemeColors,
    c,
    print_unified_result,
    print_clear,
    get_system_info,
    get_footer,
)

_last_search_data = {}
_tui_app = None


def save_markdown_report(
    query: str,
    summary: str,
    articles: list,
    keywords: list,
    mode: str,
    scraped_content: str = "",
) -> str:
    """Save research report as markdown file with FULL article text."""
    output_dir = Path.home() / "Documents" / "ACHEM"
    output_dir.mkdir(parents=True, exist_ok=True)

    safe_query = "".join(ch for ch in query if ch.isalnum() or ch in " -").replace(
        " ", "_"
    )[:50]
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    filepath = output_dir / f"{safe_query}_{timestamp}.md"

    md_content = f"""# {query}

**Generated:** {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}
**Total Sources:** {len(articles)}
**Mode:** {mode.upper()}

---

## AI Conclusion

{summary}

---

## Keywords

{", ".join(keywords[:15])}

---

## Articles Content

"""
    for i, article in enumerate(articles, 1):
        title = article.get("title", "Unknown")
        body = article.get("body", article.get("summary", ""))

        md_content += f"""### Article {i}: {title}

{body if body else "No content available."}

---

"""

    if scraped_content:
        md_content += f"""## Extracted Web Content

{scraped_content}

"""

    md_content += f"""---

*Generated by ACHEM - Deep Web Research Tool*
"""

    filepath.write_text(md_content, encoding="utf-8")
    return str(filepath)


def detect_language(text: str) -> str:
    french_chars_set = set("àâäéèêëïîôùûüçÀÂÄÉÈÊËÏÎÔÙÛÜÇ")
    arabic_chars = sum(1 for ch in text if "\u0600" <= ch <= "\u06ff")
    french_chars = sum(1 for ch in text if ch in french_chars_set)
    total = len(text)
    if total == 0:
        return "en"
    if arabic_chars / total > 0.15:
        return "ar"
    if french_chars / total > 0.05:
        return "fr"
    return "en"


def show_loading(console_obj, status: str, sources: list = None):
    """Progress output with spinner and details."""
    print_clear()

    spinner = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]
    import time

    frame = int(time.time() * 10) % len(spinner)

    console_obj.print()
    console_obj.print(
        f"  [{c('accent_blue')}]{spinner[frame]} {status}[/{c('accent_blue')}]"
    )

    if sources:
        console_obj.print(f"  [{c('yellow')}]┄┄┄┄┄┄┄┄┄┄┄┄┄┄[/{c('yellow')}]")
        for s in sources[:8]:
            console_obj.print(
                f"  [{c('green')}]├[/{c('green')}] [{c('overlay0')}]{s[:55]}[/{c('overlay0')}]"
            )
        if len(sources) > 8:
            console_obj.print(
                f"  [{c('green')}]├[/{c('green')}] [{c('overlay0')}]... +{len(sources) - 8} more[/{c('overlay0')}]"
            )

    console_obj.print(f"  [{c('red')}]Ctrl+C[/{c('red')}] to cancel")
    console_obj.print()


def generate_local_summary(articles: list, query: str, keywords: list) -> str:
    """Generate synthesis using extractive algorithm - combines and summarizes content."""
    if not articles:
        return "No articles to summarize."

    analyzer = TextAnalyzer()

    all_texts = []
    for article in articles:
        text = article.get("body", "") or article.get("summary", "")
        if text and len(text) > 30:
            all_texts.append(text)

    if not all_texts:
        for a in articles[:10]:
            s = a.get("summary", a.get("title", ""))
            if s:
                all_texts.append(s)

    if not all_texts:
        key_topics = ", ".join([k for k, _ in keywords[:5]])
        return f"Based on {len(articles)} sources about '{query}'.\n\n**Key Topics:** {key_topics}"

    combined = " ".join(all_texts)
    sentences = re.split(r"[.!?\n]+", combined)

    keyword_set = {kw.lower() for kw, _ in keywords[:15]}

    scored_sentences = []
    seen_content = set()

    for sent in sentences:
        sent = sent.strip()
        if len(sent) < 40 or len(sent) > 500:
            continue

        sent_lower = sent.lower()
        content_hash = "".join(c for c in sent_lower if c.isalnum())[:50]
        if content_hash in seen_content:
            continue
        seen_content.add(content_hash)

        score = 0
        words = set(analyzer.tokenize(sent_lower))

        for kw in keyword_set:
            if kw in sent_lower:
                score += 5
            if kw in words:
                score += 3

        score += len(sent) / 100

        if score > 0:
            scored_sentences.append((sent, score))

    scored_sentences.sort(key=lambda x: x[1], reverse=True)

    top_sents = [s for s, _ in scored_sentences[:15]]

    theme_words = [kw for kw, _ in keywords[:8]]

    conclusion_parts = []

    intro = f"Based on analysis of {len(articles)} sources, here are the key findings about '{query}':"
    conclusion_parts.append(intro)

    conclusion_parts.append("")

    paragraph = []
    for sent in top_sents[:8]:
        clean_sent = sent.strip()
        if clean_sent:
            paragraph.append(clean_sent)
            if len(paragraph) >= 3:
                break

    if paragraph:
        conclusion_parts.append(" ".join(paragraph) + ".")

    themes_para = []
    for sent in top_sents[8:]:
        clean_sent = sent.strip()
        if clean_sent and any(kw in clean_sent.lower() for kw in theme_words[:4]):
            themes_para.append(clean_sent)
            if len(themes_para) >= 2:
                break

    if themes_para:
        conclusion_parts.append("")
        conclusion_parts.append(" ".join(themes_para) + ".")

    key_topics = ", ".join([k for k, _ in keywords[:6]])

    conclusion_parts.append("")
    conclusion_parts.append(f"**Key Topics:** {key_topics}")
    conclusion_parts.append(f"**Sources Analyzed:** {len(articles)}")

    return "\n".join(conclusion_parts)


TRUSTED_SOURCES = [
    "wikipedia.org",
    "wikimedia.org",
    "cnn.com",
    "bbc.com",
    "reuters.com",
    "apnews.com",
    "nytimes.com",
    "theguardian.com",
    "nasa.gov",
    "nih.gov",
    "edu",
    "gov",
]


def is_trusted_source(url: str) -> bool:
    """Check if URL is from a trusted source."""
    url_lower = url.lower()
    return any(src in url_lower for src in TRUSTED_SOURCES)


def process_deep_search(terms: list[str], theme, args, show_loading_ui=False):
    """Deep Web Research - local or AI mode."""
    mode = getattr(args, "mode", "ai")
    ddg_limit = args.ddg_limit if args.ddg_limit else 100
    scrape_top = min(100, ddg_limit)
    query = " ".join(terms)
    start_time = time.time()

    cache = None
    if not args.no_cache:
        cache = get_sqlite_cache(ttl_seconds=args.cache_ttl)

    show_loading(
        console,
        f"[{c('green')}]LOCAL[/{c('green')}] Searching: {query[:35]}..."
        if mode == "local"
        else f"Searching: {query[:40]}...",
    )

    ddg_client = get_ddg_client(max_results=ddg_limit)

    show_loading(console, "Searching DuckDuckGo...", [])
    ddg_results = ddg_client.search(query, max_results=ddg_limit)

    sources = [r.get("title", "Unknown")[:50] for r in ddg_results[:8]]
    show_loading(
        console, f"Found {len(ddg_results)} results. Scraping content...", sources
    )

    from .web_scraper import get_scraper

    scraper = get_scraper()

    urls_to_scrape = [
        r.get("url", "") for r in ddg_results[:scrape_top] if r.get("url")
    ]
    scraped_results = scraper.scrape_batch(urls_to_scrape)

    show_loading(
        console, f"Scraped {len(scraped_results)} pages. Processing content...", []
    )

    raw_scraped_content = ""
    if scraped_results:
        for item in scraped_results:
            raw_scraped_content += (
                f"\n\n=== {item['title']} ===\n{item['content'][:3000]}"
            )

    show_loading(
        console, f"Processing {len(scraped_results)} pages with deduplication...", []
    )
    time.sleep(0.2)

    scraped_urls = {item["url"]: item["content"] for item in scraped_results}

    articles = []
    for r in ddg_results:
        url = r.get("url", "")
        is_trusted = is_trusted_source(url)
        scraped_text = scraped_urls.get(url, "")

        summary = r.get("body", "") or scraped_text[:2000]
        body = scraped_text if scraped_text else r.get("body", "")

        articles.append(
            {
                "title": r.get("title", "Unknown"),
                "summary": summary,
                "body": body,
                "url": url,
                "source": "trusted" if is_trusted else "duckduckgo",
                "relevance_score": 70
                if is_trusted
                else 50 + r.get("priority_score", 0),
            }
        )

    if not args.no_wikipedia:
        show_loading(console, "Adding Wikipedia references...", [])
        wiki_client = WikipediaClient(use_cache=False)
        wiki_articles = wiki_client.get_articles_for_queries(terms, limit_per_query=5)
        for article in wiki_articles:
            article["source"] = "wikipedia"
            article["relevance_score"] = 80
        articles.extend(wiki_articles)
        time.sleep(0.2)

    if mode == "local":
        show_loading(console, "Filtering trusted sources...", [])
        articles = [a for a in articles if a["source"] in ("wikipedia", "trusted")]
        time.sleep(0.2)

    show_loading(console, "Filtering by context...", [])
    articles, filtered = classify_and_filter(articles, query, threshold=15)
    if filtered:
        console.print(
            f"  [{c('overlay0')}]Filtered {len(filtered)} noise sources[/{c('overlay0')}]"
        )

    show_loading(console, "Calculating Ordinal Distance...", [])
    positional_ranker = PositionalRanker()
    target_word = positional_ranker.extract_target_word(query)
    if target_word:
        from .positional_ranker import OrdinalDistanceCalculator

        calc = OrdinalDistanceCalculator()
        query_pattern = calc.calculate_ordinal_pattern(query, target_word)

        articles = positional_ranker.rank_results(articles, query)

        if articles:
            avg_dist = sum(a.get("ordinal_distance", 0) for a in articles) / len(
                articles
            )
            min_dist = min(a.get("ordinal_distance", 999) for a in articles)
            best = articles[0]
            best_dist = best.get("ordinal_distance", 0)

            console.print(
                f"  [{c('accent_blue')}]Target: '[bold]{target_word}[/bold]' @ pos {query_pattern.target_position + 1}[/{c('accent_blue')}]"
            )
            console.print(
                f"  [{c('green')}]Best match: pos {best.get('title_target_pos', 0) + 1}, dist={best_dist:.0f}[/{c('green')}]"
            )
    time.sleep(0.1)

    show_loading(console, f"Analyzing {len(articles)} sources...", [])
    analyzer = TextAnalyzer()
    articles = analyzer.score_articles_with_relevance(articles, terms)
    time.sleep(0.2)

    if args.min_relevance > 0:
        articles, removed = analyzer.filter_by_relevance(articles, args.min_relevance)
        if removed:
            console.print(
                f"[{c('overlay0')}]Filtered {len(removed)} results below {args.min_relevance}% relevance[/{c('overlay0')}]\n"
            )

    if not articles:
        console.print(f"[{c('yellow')}]No relevant results found.[/{c('yellow')}]\n")
        return

    keywords = analyzer.find_recurring_keywords(articles, top_n=15)
    final_count = len(articles)
    keyword_list = [kw for kw, _ in keywords]
    query_words = set(query.lower().split())
    all_keywords = keyword_list + list(query_words)

    show_loading(
        console,
        f"Processing {len(scraped_results)} pages with keyword filtering...",
        [],
    )

    def filter_relevant(text: str, keywords: list, min_matches: int = 1) -> str:
        if not text:
            return ""
        sentences = re.split(r"[.!?\n]+", text)
        relevant = []
        for sent in sentences:
            sent = sent.strip()
            if len(sent) < 30:
                continue
            matches = sum(1 for kw in keywords if kw.lower() in sent.lower())
            if matches >= min_matches:
                relevant.append(sent)
        return ". ".join(relevant)

    scraped_content = ""
    if scraped_results:
        filtered_parts = []
        for item in scraped_results:
            title = item.get("title", "")
            content = item.get("content", "")
            if content and len(content) > 100:
                filtered = filter_relevant(content, all_keywords, min_matches=1)
                if filtered:
                    filtered_parts.append(f"[{title}]\n{filtered}")
        scraped_content = "\n\n".join(filtered_parts)

    if not scraped_content and scraped_results:
        scraped_content = "\n\n".join(
            [
                f"[{item.get('title', '')}]\n{item.get('content', '')[:2000]}"
                for item in scraped_results[:30]
                if item.get("content")
            ]
        )

    combined_text = " ".join([a.get("summary", "") for a in articles])
    lang = detect_language(combined_text) if args.lang == "auto" else args.lang

    if mode == "local":
        show_loading(console, "Generating summary (TF-IDF)...", [])
        unified_summary = generate_local_summary(articles, query, keywords)
        result_mode = "local"
    else:
        show_loading(console, "Generating AI summary...", [])
        config = ConfigManager()
        openrouter_summarizer = get_openrouter_summarizer()
        groq_summarizer = get_groq_summarizer()
        gemini_summarizer = get_gemini_summarizer()
        local_summarizer = get_local_summarizer(config)

        if openrouter_summarizer.is_ai_available():
            unified_summary, result_mode = (
                openrouter_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif groq_summarizer.is_ai_available():
            unified_summary, result_mode = (
                groq_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif gemini_summarizer.is_ai_available():
            unified_summary, result_mode = (
                gemini_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif local_summarizer.is_ai_available():
            unified_summary, result_mode = (
                local_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif groq_summarizer.is_ai_available():
            unified_summary, result_mode = (
                groq_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif gemini_summarizer.is_ai_available():
            unified_summary, result_mode = (
                gemini_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        elif local_summarizer.is_ai_available():
            unified_summary, result_mode = (
                local_summarizer.generate_deep_research_summary(
                    search_snippets=articles,
                    scraped_content=scraped_content,
                    language=lang,
                    query=query,
                )
            )
        else:
            show_loading(console, "Generating summary (TF-IDF)...", [])
            unified_summary = generate_local_summary(articles, query, keywords)
            result_mode = "local"

    show_loading(console, "Finalizing results...", [])
    time.sleep(0.2)

    mode_label = f"[bold {c('green')}]{mode.upper()}[/bold {c('green')}]"

    print_clear()
    print_unified_result(
        articles,
        [kw for kw, _ in keywords],
        unified_summary,
        theme,
        result_mode,
        source_count=final_count,
    )

    global _last_search_data
    _last_search_data = {
        "summary": unified_summary,
        "query": query,
        "keywords": [kw for kw, _ in keywords],
        "source_count": final_count,
    }

    if cache:
        cache.set(
            query=query,
            articles=articles,
            summary=unified_summary,
            summary_mode=result_mode,
            relevance_scores={a["title"]: a["relevance_score"] for a in articles},
            keywords=[kw for kw, _ in keywords],
            full_content=scraped_content,
        )

    markdown_path = save_markdown_report(
        query,
        unified_summary,
        articles,
        [kw for kw, _ in keywords],
        result_mode,
        scraped_content,
    )
    console.print(
        f"[{c('overlay0')}]Report saved to: {markdown_path}[/{c('overlay0')}]\n"
    )

    elapsed = time.time() - start_time
    sys_info = get_system_info()
    cache_stats = cache.get_stats() if cache else {}
    console.print(
        get_footer(sys_info["cpu_percent"], sys_info["memory_mb"], cache_stats)
    )
    console.print(
        f"\n[{c('overlay0')}]Research completed in {round(elapsed, 1)}s ({mode_label})[/{c('overlay0')}]\n"
    )


def handle_followup(question: str, cached_data: dict, mode: str) -> str:
    """Handle follow-up question using cached data."""
    articles = cached_data.get("articles", [])
    full_content = cached_data.get("full_content", "")
    keywords = cached_data.get("keywords", [])

    combined_context = ""

    for article in articles[:10]:
        title = article.get("title", "")
        body = article.get("body", article.get("summary", ""))
        if body:
            combined_context += f"- {body[:300]}\n"

    if full_content:
        combined_context += f"\n--- Extended Content ---\n{full_content[:3000]}"

    if not combined_context:
        return "No additional context available for follow-up questions."

    if mode == "ai":
        try:
            groq = get_groq_summarizer()
            openrouter = get_openrouter_summarizer()
            gemini = get_gemini_summarizer()

            if groq.is_ai_available():
                summarizer = groq
            elif openrouter.is_ai_available():
                summarizer = openrouter
            elif gemini.is_ai_available():
                summarizer = gemini
            else:
                return generate_local_followup(question, articles, keywords)

            answer, _ = summarizer.generate_deep_research_summary(
                search_snippets=articles,
                scraped_content=combined_context,
                language="en",
                query=question,
            )
            return answer
        except Exception:
            pass

    return generate_local_followup(question, articles, keywords)


def generate_local_followup(question: str, articles: list, keywords: list) -> str:
    """Generate follow-up answer using local TF-IDF."""
    question_lower = question.lower()
    question_words = set(re.findall(r"\w+", question_lower))

    relevant_parts = []

    for article in articles:
        body = article.get("body", article.get("summary", ""))
        if not body:
            continue

        score = 0
        for word in question_words:
            if len(word) < 3:
                continue
            if word in body.lower():
                score += 5

        if score > 0:
            sentences = re.split(r"[.!?]+", body)
            for sent in sentences:
                sent = sent.strip()
                if len(sent) < 20:
                    continue
                if any(w in sent.lower() for w in question_words if len(w) > 2):
                    relevant_parts.append((sent, score))

    relevant_parts.sort(key=lambda x: x[1], reverse=True)

    if not relevant_parts:
        return "I couldn't find specific information about that in the sources."

    answer_parts = [p[0].capitalize() + "." for p, _ in relevant_parts[:3]]
    return " ".join(answer_parts)


def main():
    parser = argparse.ArgumentParser(description="ACHEM - Deep Web Research Tool")
    parser.add_argument("queries", nargs="*", help="Search terms")
    parser.add_argument("-l", "--limit", type=int, default=10)
    parser.add_argument(
        "--mode",
        choices=["local", "ai"],
        default="ai",
        help="local: TF-IDF, ai: cloud AI",
    )
    parser.add_argument("--lang", choices=["en", "fr", "ar", "auto"], default="auto")
    parser.add_argument("--no-spell-check", action="store_true")
    parser.add_argument("--no-filter", action="store_true")
    parser.add_argument("--no-cache", action="store_true")
    parser.add_argument("--no-wikipedia", action="store_true")
    parser.add_argument("--ddg-limit", type=int, default=100)
    parser.add_argument("--clear-cache", action="store_true")
    parser.add_argument("--cache-ttl", type=int, default=86400)
    parser.add_argument("--min-relevance", type=float, default=0)

    args = parser.parse_args()

    if args.queries:
        terms = []
        for q in args.queries:
            terms.extend([t.strip() for t in q.split(",") if t.strip()])
        theme = detect_theme(" ".join(terms)) if terms else ThemeColors.DEFAULT
        process_deep_search(terms, theme, args)
        return

    from .tui import run_tui as _run_tui

    while True:
        result = _run_tui()

        if result is None:
            console.print(f"\n[{c('accent_blue')}]Goodbye![/{c('accent_blue')}]\n")
            break

        if result:
            if isinstance(result, tuple):
                query_str, mode = result
            else:
                query_str = result
                mode = "ai"

            terms = [t.strip() for t in query_str.split(",") if t.strip()]
            if terms:
                theme = detect_theme(terms[0])
                args.mode = mode
                process_deep_search(terms, theme, args)

                current_query = query_str
                current_mode = mode

                while True:
                    console.print()
                    followup = console.input(
                        f"[{c('accent_blue')}]? [{c('yellow')}]Ask follow-up[/{c('yellow')}] "
                        f"[{c('overlay0')}]or Ctrl+C new search: [/{c('overlay0')}]"
                    )

                    if not followup.strip():
                        break

                    if followup.strip().lower() in ["q", "quit", "exit"]:
                        break

                    if followup.strip().startswith(":"):
                        break

                    show_loading(console, f"Finding answer...", [])

                    cache = get_sqlite_cache()
                    cached = cache.get(current_query)

                    if cached:
                        answer = handle_followup(followup, cached, current_mode)
                        print_clear()
                        console.print(f"\n[{c('green')}]▸ {followup}[/{c('green')}]\n")
                        console.print(f"[{c('text')}]{answer}[/{c('text')}]\n")
                    else:
                        console.print(
                            f"[{c('yellow')}]Context expired. Please search again.[/{c('yellow')}]\n"
                        )
                        break


if __name__ == "__main__":
    main()
