import os
import json
import time
import sqlite3
import threading
from datetime import datetime
from flask import Flask, render_template, request, jsonify, send_from_directory, Response, make_response, current_app, Blueprint, redirect, url_for, flash
from flask_socketio import SocketIO, emit
from local_deep_research.search_system import AdvancedSearchSystem
from local_deep_research.report_generator import IntegratedReportGenerator
# Move this import up to ensure it's available globally
from dateutil import parser
import traceback
import pkg_resources
# Import the new configuration manager
from local_deep_research.config import get_config_dir 
import logging
logger = logging.getLogger(__name__)

CONFIG_DIR = get_config_dir() / "config"
MAIN_CONFIG_FILE = CONFIG_DIR / "settings.toml"
LLM_CONFIG_FILE = CONFIG_DIR / "llm_config.py"
LOCAL_COLLECTIONS_FILE = CONFIG_DIR / "local_collections.toml"
import toml

# Set flag for tracking OpenAI availability - we'll check it only when needed
OPENAI_AVAILABLE = False

# Initialize Flask app
try:
    import os
    import logging
    from local_deep_research.utilties.setup_utils import setup_user_directories
    
    # Configure logging
    logging.basicConfig(level=logging.INFO)
    logger = logging.getLogger(__name__)
    
    # Explicitly run setup
    logger.info("Initializing configuration...")
    setup_user_directories()
    
    # Get directories based on package installation
    PACKAGE_DIR = pkg_resources.resource_filename('local_deep_research', 'web')
    STATIC_DIR = os.path.join(PACKAGE_DIR, 'static')
    TEMPLATE_DIR = os.path.join(PACKAGE_DIR, 'templates')
    
    # Setup logging
    logging.basicConfig(level=logging.INFO)
    
    # Create directories and default configs if needed
    setup_user_directories()

    # Initialize Flask app with package directories
    app = Flask(__name__, 
                static_folder=STATIC_DIR,
                template_folder=TEMPLATE_DIR)
    print(f"Using package static path: {STATIC_DIR}")
    print(f"Using package template path: {TEMPLATE_DIR}")
except Exception as e:
    # Fallback for development
    print(f"Package directories not found, using fallback paths: {str(e)}")
    app = Flask(__name__, 
                static_folder=os.path.abspath('static'),
                template_folder=os.path.abspath('templates'))
app.config['SECRET_KEY'] = 'deep-research-secret-key'

# Create a Blueprint for the research application
research_bp = Blueprint('research', __name__, url_prefix='/research')

# Add improved Socket.IO configuration with better error handling
socketio = SocketIO(
    app, 
    cors_allowed_origins="*",
    async_mode='threading',
    path='/research/socket.io',
    logger=True,
    engineio_logger=True,
    ping_timeout=20,
    ping_interval=5
)

# Active research processes and socket subscriptions
active_research = {}
socket_subscriptions = {}

# Add termination flags dictionary
termination_flags = {}

# Database setup
DB_PATH = 'research_history.db'

# Output directory for research results
OUTPUT_DIR = 'research_outputs'

# Add Content Security Policy headers to allow Socket.IO to function
@app.after_request
def add_security_headers(response):
    # Define a permissive CSP for development that allows Socket.IO to function
    csp = (
        "default-src 'self'; "
        "connect-src 'self' ws: wss: http: https:; " 
        "script-src 'self' 'unsafe-inline' 'unsafe-eval' cdnjs.cloudflare.com cdn.jsdelivr.net unpkg.com; "
        "style-src 'self' 'unsafe-inline' cdnjs.cloudflare.com; "
        "font-src 'self' cdnjs.cloudflare.com; "
        "img-src 'self' data:; "
        "worker-src blob:; "
        "frame-src 'self';"
    )
    
    response.headers['Content-Security-Policy'] = csp
    response.headers['X-Content-Security-Policy'] = csp
    
    # Add CORS headers for API requests
    if request.path.startswith('/api/'):
        response.headers['Access-Control-Allow-Origin'] = '*'
        response.headers['Access-Control-Allow-Methods'] = 'GET, POST, DELETE, OPTIONS'
        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
    
    return response

# Add a middleware layer to handle abrupt disconnections
@app.before_request
def handle_websocket_requests():
    if request.path.startswith('/research/socket.io'):
        try:
            if not request.environ.get('werkzeug.socket'):
                return
        except Exception as e:
            print(f"WebSocket preprocessing error: {e}")
            # Return empty response to prevent further processing
            return '', 200

def init_db():
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    
    # Create the table if it doesn't exist
    cursor.execute('''
    CREATE TABLE IF NOT EXISTS research_history (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        query TEXT NOT NULL,
        mode TEXT NOT NULL,
        status TEXT NOT NULL,
        created_at TEXT NOT NULL,
        completed_at TEXT,
        duration_seconds INTEGER,
        report_path TEXT,
        metadata TEXT,
        progress_log TEXT,
        progress INTEGER
    )
    ''')
    
    # Create a dedicated table for research logs
    cursor.execute('''
    CREATE TABLE IF NOT EXISTS research_logs (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        research_id INTEGER NOT NULL,
        timestamp TEXT NOT NULL,
        message TEXT NOT NULL,
        log_type TEXT NOT NULL,
        progress INTEGER,
        metadata TEXT,
        FOREIGN KEY (research_id) REFERENCES research_history (id) ON DELETE CASCADE
    )
    ''')
    
    # Check if the duration_seconds column exists, add it if missing
    cursor.execute('PRAGMA table_info(research_history)')
    columns = [column[1] for column in cursor.fetchall()]
    
    if 'duration_seconds' not in columns:
        print("Adding missing 'duration_seconds' column to research_history table")
        cursor.execute('ALTER TABLE research_history ADD COLUMN duration_seconds INTEGER')
    
    # Check if the progress column exists, add it if missing
    if 'progress' not in columns:
        print("Adding missing 'progress' column to research_history table")
        cursor.execute('ALTER TABLE research_history ADD COLUMN progress INTEGER')
    
    # Enable foreign key support
    cursor.execute('PRAGMA foreign_keys = ON')
    
    conn.commit()
    conn.close()
    
# Helper function to calculate duration between created_at and completed_at timestamps
def calculate_duration(created_at_str):
    """
    Calculate duration in seconds between created_at timestamp and now.
    Handles various timestamp formats and returns None if calculation fails.
    """
    if not created_at_str:
        return None
        
    now = datetime.utcnow()
    duration_seconds = None
    
    try:
        # Proper parsing of ISO format
        if 'T' in created_at_str:  # ISO format with T separator
            start_time = datetime.fromisoformat(created_at_str)
        else:  # Older format without T
            # Try different formats
            try:
                start_time = datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S.%f')
            except ValueError:
                try:
                    start_time = datetime.strptime(created_at_str, '%Y-%m-%d %H:%M:%S')
                except ValueError:
                    # Last resort fallback
                    start_time = datetime.fromisoformat(created_at_str.replace(' ', 'T'))
        
        # Ensure we're comparing UTC times
        duration_seconds = int((now - start_time).total_seconds())
    except Exception as e:
        print(f"Error calculating duration: {str(e)}")
        # Fallback method if parsing fails
        try:
            start_time_fallback = parser.parse(created_at_str)
            duration_seconds = int((now - start_time_fallback).total_seconds())
        except:
            print(f"Fallback duration calculation also failed for timestamp: {created_at_str}")
    
    return duration_seconds

# Add these helper functions after the calculate_duration function


def add_log_to_db(research_id, message, log_type='info', progress=None, metadata=None):
    """
    Store a log entry in the database
    
    Args:
        research_id: ID of the research
        message: Log message text
        log_type: Type of log (info, error, milestone)
        progress: Progress percentage (0-100)
        metadata: Additional metadata as dictionary (will be stored as JSON)
    """
    try:
        timestamp = datetime.utcnow().isoformat()
        metadata_json = json.dumps(metadata) if metadata else None
        
        conn = sqlite3.connect(DB_PATH)
        cursor = conn.cursor()
        cursor.execute(
            'INSERT INTO research_logs (research_id, timestamp, message, log_type, progress, metadata) '
            'VALUES (?, ?, ?, ?, ?, ?)',
            (research_id, timestamp, message, log_type, progress, metadata_json)
        )
        conn.commit()
        conn.close()
        return True
    except Exception as e:
        print(f"Error adding log to database: {str(e)}")
        print(traceback.format_exc())
        return False

def get_logs_for_research(research_id):
    """
    Retrieve all logs for a specific research ID
    
    Args:
        research_id: ID of the research
    
    Returns:
        List of log entries as dictionaries
    """
    try:
        conn = sqlite3.connect(DB_PATH)
        conn.row_factory = sqlite3.Row
        cursor = conn.cursor()
        cursor.execute(
            'SELECT * FROM research_logs WHERE research_id = ? ORDER BY timestamp ASC',
            (research_id,)
        )
        results = cursor.fetchall()
        conn.close()
        
        logs = []
        for result in results:
            log_entry = dict(result)
            # Parse metadata JSON if it exists
            if log_entry.get('metadata'):
                try:
                    log_entry['metadata'] = json.loads(log_entry['metadata'])
                except:
                    log_entry['metadata'] = {}
            else:
                log_entry['metadata'] = {}
            
            # Convert entry for frontend consumption
            formatted_entry = {
                'time': log_entry['timestamp'],
                'message': log_entry['message'],
                'progress': log_entry['progress'],
                'metadata': log_entry['metadata'],
                'type': log_entry['log_type']
            }
            logs.append(formatted_entry)
            
        return logs
    except Exception as e:
        print(f"Error retrieving logs from database: {str(e)}")
        print(traceback.format_exc())
        return []
    
# Initialize the database on startup
def initialize():
    init_db()

# Call initialize immediately when app is created
initialize()

# Route for index page - keep this at root level for easy access
@app.route('/')
def root_index():
    return redirect(url_for('research.index'))

# Update all routes with the research prefix
@research_bp.route('/')
def index():
    return render_template('index.html')

@research_bp.route('/static/<path:path>')
def serve_static(path):
    try:
        print(f"Serving static file: {path}")
        print(f"Static folder path: {app.static_folder}")
        return send_from_directory(app.static_folder, path)
    except Exception as e:
        print(f"Error serving static file {path}: {str(e)}")
        return f"Error serving file: {str(e)}", 404

@research_bp.route('/api/history', methods=['GET'])
def get_history():
    """Get the research history"""
    try:
        conn = sqlite3.connect(DB_PATH)
        conn.row_factory = sqlite3.Row
        cursor = conn.cursor()
        
        # Get all history records ordered by latest first
        cursor.execute('SELECT * FROM research_history ORDER BY created_at DESC')
        results = cursor.fetchall()
        conn.close()
        
        # Convert to list of dicts
        history = []
        for result in results:
            item = dict(result)
            
            # Ensure all keys exist with default values
            if 'id' not in item:
                item['id'] = None
            if 'query' not in item:
                item['query'] = 'Untitled Research'
            if 'mode' not in item:
                item['mode'] = 'quick'
            if 'status' not in item:
                item['status'] = 'unknown'
            if 'created_at' not in item:
                item['created_at'] = None
            if 'completed_at' not in item:
                item['completed_at'] = None
            if 'duration_seconds' not in item:
                item['duration_seconds'] = None
            if 'report_path' not in item:
                item['report_path'] = None
            if 'metadata' not in item:
                item['metadata'] = '{}'
            if 'progress_log' not in item:
                item['progress_log'] = '[]'
            
            # Ensure timestamps are in ISO format
            if item['created_at'] and 'T' not in item['created_at']:
                try:
                    # Convert to ISO format if it's not already
                    dt = parser.parse(item['created_at'])
                    item['created_at'] = dt.isoformat()
                except:
                    pass
                
            if item['completed_at'] and 'T' not in item['completed_at']:
                try:
                    # Convert to ISO format if it's not already
                    dt = parser.parse(item['completed_at'])
                    item['completed_at'] = dt.isoformat()
                except:
                    pass
                
            # Recalculate duration based on timestamps if it's null but both timestamps exist
            if item['duration_seconds'] is None and item['created_at'] and item['completed_at']:
                try:
                    start_time = parser.parse(item['created_at'])
                    end_time = parser.parse(item['completed_at'])
                    item['duration_seconds'] = int((end_time - start_time).total_seconds())
                except Exception as e:
                    print(f"Error recalculating duration: {str(e)}")
            
            history.append(item)
        
        # Add CORS headers
        response = make_response(jsonify(history))
        response.headers.add('Access-Control-Allow-Origin', '*')
        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
        return response
    except Exception as e:
        print(f"Error getting history: {str(e)}")
        print(traceback.format_exc())
        # Return empty array with CORS headers
        response = make_response(jsonify([]))
        response.headers.add('Access-Control-Allow-Origin', '*')
        response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
        response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
        return response

@research_bp.route('/api/start_research', methods=['POST'])
def start_research():
    data = request.json
    query = data.get('query')
    mode = data.get('mode', 'quick')
    
    if not query:
        return jsonify({'status': 'error', 'message': 'Query is required'}), 400
        
    # Check if there's any active research that's actually still running
    if active_research:
        # Verify each active research is still valid
        stale_research_ids = []
        for research_id, research_data in list(active_research.items()):
            # Check database status
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            cursor.execute('SELECT status FROM research_history WHERE id = ?', (research_id,))
            result = cursor.fetchone()
            conn.close()
            
            # If the research doesn't exist in DB or is not in_progress, it's stale
            if not result or result[0] != 'in_progress':
                stale_research_ids.append(research_id)
            # Also check if thread is still alive
            elif not research_data.get('thread') or not research_data.get('thread').is_alive():
                stale_research_ids.append(research_id)

        # Clean up any stale research processes
        for stale_id in stale_research_ids:
            print(f"Cleaning up stale research process: {stale_id}")
            if stale_id in active_research:
                del active_research[stale_id]
            if stale_id in termination_flags:
                del termination_flags[stale_id]

        # After cleanup, check if there's still active research
        if active_research:
            return jsonify({
                'status': 'error', 
                'message': 'Another research is already in progress. Please wait for it to complete.'
            }), 409
        
    # Create a record in the database with explicit UTC timestamp
    created_at = datetime.utcnow().isoformat()
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    cursor.execute(
        'INSERT INTO research_history (query, mode, status, created_at, progress_log) VALUES (?, ?, ?, ?, ?)',
        (query, mode, 'in_progress', created_at, json.dumps([{"time": created_at, "message": "Research started", "progress": 0}]))
    )
    research_id = cursor.lastrowid
    conn.commit()
    conn.close()
    
    # Start research process in a background thread
    thread = threading.Thread(
        target=run_research_process,
        args=(research_id, query, mode)
    )
    thread.daemon = True
    thread.start()
    
    active_research[research_id] = {
        'thread': thread,
        'progress': 0,
        'status': 'in_progress',
        'log': [{"time": created_at, "message": "Research started", "progress": 0}]
    }
    
    return jsonify({
        'status': 'success',
        'research_id': research_id
    })

@research_bp.route('/api/research/<int:research_id>')
def get_research_status(research_id):
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()
    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
    result = dict(cursor.fetchone() or {})
    conn.close()
    
    if not result:
        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
        
    # Add progress information
    if research_id in active_research:
        result['progress'] = active_research[research_id]['progress']
        result['log'] = active_research[research_id]['log']
    elif result.get('status') == 'completed':
        result['progress'] = 100
        try:
            result['log'] = json.loads(result.get('progress_log', '[]'))
        except:
            result['log'] = []
    else:
        result['progress'] = 0
        try:
            result['log'] = json.loads(result.get('progress_log', '[]'))
        except:
            result['log'] = []
        
    return jsonify(result)

@research_bp.route('/api/research/<int:research_id>/details')
def get_research_details(research_id):
    """Get detailed progress log for a specific research"""
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()
    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
    result = dict(cursor.fetchone() or {})
    conn.close()
    
    if not result:
        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
    
    # Get logs from the dedicated log database
    logs = get_logs_for_research(research_id)
    
    # If this is an active research, merge with any in-memory logs
    if research_id in active_research:
        # Use the logs from memory temporarily until they're saved to the database
        memory_logs = active_research[research_id]['log']
        
        # Filter out logs that are already in the database by timestamp
        db_timestamps = {log['time'] for log in logs}
        unique_memory_logs = [log for log in memory_logs if log['time'] not in db_timestamps]
        
        # Add unique memory logs to our return list
        logs.extend(unique_memory_logs)
        
        # Sort logs by timestamp
        logs.sort(key=lambda x: x['time'])
    
    return jsonify({
        'status': 'success',
        'research_id': research_id,
        'query': result.get('query'),
        'mode': result.get('mode'),
        'status': result.get('status'),
        'progress': active_research.get(research_id, {}).get('progress', 100 if result.get('status') == 'completed' else 0),
        'created_at': result.get('created_at'),
        'completed_at': result.get('completed_at'),
        'log': logs
    })

@research_bp.route('/api/report/<int:research_id>')
def get_report(research_id):
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()
    cursor.execute('SELECT * FROM research_history WHERE id = ?', (research_id,))
    result = dict(cursor.fetchone() or {})
    conn.close()
    
    if not result or not result.get('report_path'):
        return jsonify({'status': 'error', 'message': 'Report not found'}), 404
        
    try:
        with open(result['report_path'], 'r', encoding='utf-8') as f:
            content = f.read()
        return jsonify({
            'status': 'success',
            'content': content,
            'metadata': json.loads(result.get('metadata', '{}'))
        })
    except Exception as e:
        return jsonify({'status': 'error', 'message': str(e)}), 500

@research_bp.route('/research/details/<int:research_id>')
def research_details_page(research_id):
    """Render the research details page"""
    return render_template('index.html')

@socketio.on('connect')
def handle_connect():
    print(f"Client connected: {request.sid}")

@socketio.on('disconnect')
def handle_disconnect():
    try:
        print(f"Client disconnected: {request.sid}")
        # Clean up subscriptions for this client
        for research_id, subscribers in list(socket_subscriptions.items()):
            if request.sid in subscribers:
                subscribers.remove(request.sid)
            if not subscribers:
                socket_subscriptions.pop(research_id, None)
                print(f"Removed empty subscription for research {research_id}")
    except Exception as e:
        print(f"Error handling disconnect: {e}")

@socketio.on('subscribe_to_research')
def handle_subscribe(data):
    research_id = data.get('research_id')
    if research_id:
        # First check if this research is still active
        conn = sqlite3.connect(DB_PATH)
        cursor = conn.cursor()
        cursor.execute('SELECT status FROM research_history WHERE id = ?', (research_id,))
        result = cursor.fetchone()
        conn.close()
        
        # Only allow subscription to valid research
        if result:
            status = result[0]
            
            # Initialize subscription set if needed
            if research_id not in socket_subscriptions:
                socket_subscriptions[research_id] = set()
            
            # Add this client to the subscribers
            socket_subscriptions[research_id].add(request.sid)
            print(f"Client {request.sid} subscribed to research {research_id}")
            
            # Send current status immediately if available
            if research_id in active_research:
                progress = active_research[research_id]['progress']
                latest_log = active_research[research_id]['log'][-1] if active_research[research_id]['log'] else None
                
                if latest_log:
                    emit(f'research_progress_{research_id}', {
                        'progress': progress,
                        'message': latest_log.get('message', 'Processing...'),
                        'status': 'in_progress',
                        'log_entry': latest_log
                    })
            elif status in ['completed', 'failed', 'suspended']:
                # Send final status for completed research
                emit(f'research_progress_{research_id}', {
                    'progress': 100 if status == 'completed' else 0,
                    'message': 'Research completed successfully' if status == 'completed' else 
                               'Research failed' if status == 'failed' else 'Research was suspended',
                    'status': status,
                    'log_entry': {
                        'time': datetime.utcnow().isoformat(),
                        'message': f'Research is {status}',
                        'progress': 100 if status == 'completed' else 0,
                        'metadata': {'phase': 'complete' if status == 'completed' else 'error'}
                    }
                })
        else:
            # Research not found
            emit('error', {'message': f'Research ID {research_id} not found'})

@socketio.on_error
def handle_socket_error(e):
    print(f"Socket.IO error: {str(e)}")
    # Don't propagate exceptions to avoid crashing the server
    return False

@socketio.on_error_default
def handle_default_error(e):
    print(f"Unhandled Socket.IO error: {str(e)}")
    # Don't propagate exceptions to avoid crashing the server
    return False

# Function to clean up resources for a completed research
def cleanup_research_resources(research_id):
    """Clean up resources for a completed research"""
    print(f"Cleaning up resources for research {research_id}")
    
    # Get the current status from the database to determine the final status message
    current_status = "completed"  # Default
    try:
        conn = sqlite3.connect(DB_PATH)
        cursor = conn.cursor()
        cursor.execute('SELECT status FROM research_history WHERE id = ?', (research_id,))
        result = cursor.fetchone()
        if result and result[0]:
            current_status = result[0]
        conn.close()
    except Exception as e:
        print(f"Error retrieving research status during cleanup: {e}")
    
    # Remove from active research
    if research_id in active_research:
        del active_research[research_id]
        
    # Remove from termination flags
    if research_id in termination_flags:
        del termination_flags[research_id]
    
    # Send a final message to any remaining subscribers with explicit status
    if research_id in socket_subscriptions and socket_subscriptions[research_id]:
        # Use the proper status message based on database status
        if current_status == 'suspended' or current_status == 'failed':
            final_message = {
                'status': current_status,
                'message': f'Research was {current_status}',
                'progress': 0,  # For suspended research, show 0% not 100%
            }
        else:
            final_message = {
                'status': 'completed',
                'message': 'Research process has ended and resources have been cleaned up',
                'progress': 100,
            }
        
        try:
            print(f"Sending final {current_status} socket message for research {research_id}")
            # Use emit to all, not just subscribers
            socketio.emit(f'research_progress_{research_id}', final_message)
            
            # Also emit to specific subscribers
            for sid in socket_subscriptions[research_id]:
                try:
                    socketio.emit(
                        f'research_progress_{research_id}', 
                        final_message,
                        room=sid
                    )
                except Exception as sub_err:
                    print(f"Error emitting to subscriber {sid}: {str(sub_err)}")
        except Exception as e:
            print(f"Error sending final cleanup message: {e}")
    
    # Don't immediately remove subscriptions - let clients disconnect naturally

def run_research_process(research_id, query, mode):
    """Run the research process in the background for a given research ID"""
    try:
        # Check if this research has been terminated before we even start
        if research_id in termination_flags and termination_flags[research_id]:
            print(f"Research {research_id} was terminated before starting")
            cleanup_research_resources(research_id)
            return

        print(f"Starting research process for ID {research_id}, query: {query}")
        
        # Set up the AI Context Manager
        output_dir = os.path.join(OUTPUT_DIR, f"research_{research_id}")
        os.makedirs(output_dir, exist_ok=True)

        # Set up progress callback
        def progress_callback(message, progress_percent, metadata):
            # FREQUENT TERMINATION CHECK: Check for termination at each callback
            if research_id in termination_flags and termination_flags[research_id]:
                # Explicitly set the status to suspended in the database
                conn = sqlite3.connect(DB_PATH)
                cursor = conn.cursor()
                # Calculate duration up to termination point - using UTC consistently
                now = datetime.utcnow()
                completed_at = now.isoformat()
                
                # Get the start time from the database
                cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
                result = cursor.fetchone()
                
                # Calculate the duration
                duration_seconds = calculate_duration(result[0]) if result and result[0] else None
                
                # Update the database with suspended status
                cursor.execute(
                    'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ? WHERE id = ?',
                    ('suspended', completed_at, duration_seconds, research_id)
                )
                conn.commit()
                conn.close()
                
                # Clean up resources
                cleanup_research_resources(research_id)
                
                # Raise exception to exit the process
                raise Exception("Research was terminated by user")
            
            timestamp = datetime.utcnow().isoformat()
            
            # Adjust progress based on research mode
            adjusted_progress = progress_percent
            if mode == 'detailed' and metadata.get('phase') == 'output_generation':
                # For detailed mode, we need to adjust the progress range
                # because detailed reports take longer after the search phase
                adjusted_progress = min(80, progress_percent)
            elif mode == 'detailed' and metadata.get('phase') == 'report_generation':
                # Scale the progress from 80% to 95% for the report generation phase
                # Map progress_percent values (0-100%) to the (80-95%) range
                if progress_percent is not None:
                    normalized = progress_percent / 100
                    adjusted_progress = 80 + (normalized * 15)
            elif mode == 'quick' and metadata.get('phase') == 'output_generation':
                # For quick mode, ensure we're at least at 85% during output generation
                adjusted_progress = max(85, progress_percent)
                # Map any further progress within output_generation to 85-95% range
                if progress_percent is not None and progress_percent > 0:
                    normalized = progress_percent / 100
                    adjusted_progress = 85 + (normalized * 10)
            
            # Don't let progress go backwards
            if research_id in active_research and adjusted_progress is not None:
                current_progress = active_research[research_id].get('progress', 0)
                adjusted_progress = max(current_progress, adjusted_progress)
            
            log_entry = {
                "time": timestamp,
                "message": message,
                "progress": adjusted_progress,
                "metadata": metadata
            }
            
            # Check if termination was requested
            if research_id in termination_flags and termination_flags[research_id]:
                # Explicitly set the status to suspended in the database
                conn = sqlite3.connect(DB_PATH)
                cursor = conn.cursor()
                # Calculate duration up to termination point - using UTC consistently
                now = datetime.utcnow()
                completed_at = now.isoformat()
                
                # Get the start time from the database
                cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
                result = cursor.fetchone()
                
                # Calculate the duration
                duration_seconds = calculate_duration(result[0]) if result and result[0] else None
                
                # Update the database with suspended status
                cursor.execute(
                    'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ? WHERE id = ?',
                    ('suspended', completed_at, duration_seconds, research_id)
                )
                conn.commit()
                conn.close()
                
                # Clean up resources
                cleanup_research_resources(research_id)
                
                # Raise exception to exit the process
                raise Exception("Research was terminated by user")
            
            # Update active research record
            if research_id in active_research:
                active_research[research_id]['log'].append(log_entry)
                if adjusted_progress is not None:
                    active_research[research_id]['progress'] = adjusted_progress
                
                # Determine log type for database storage
                log_type = 'info'
                if metadata and metadata.get('phase'):
                    phase = metadata.get('phase')
                    if phase in ['complete', 'iteration_complete']:
                        log_type = 'milestone'
                    elif phase == 'error' or 'error' in message.lower():
                        log_type = 'error'
                
                # Always save logs to the new research_logs table
                add_log_to_db(
                    research_id,
                    message,
                    log_type=log_type,
                    progress=adjusted_progress,
                    metadata=metadata
                )
                
                # Update progress in the research_history table (for backward compatibility)
                conn = sqlite3.connect(DB_PATH)
                cursor = conn.cursor()
                
                # Update the progress and log separately to avoid race conditions with reading/writing the log
                if adjusted_progress is not None:
                    cursor.execute(
                        'UPDATE research_history SET progress = ? WHERE id = ?',
                        (adjusted_progress, research_id)
                    )
                
                # Add the log entry to the progress_log
                cursor.execute('SELECT progress_log FROM research_history WHERE id = ?', (research_id,))
                log_result = cursor.fetchone()
                
                if log_result:
                    try:
                        current_log = json.loads(log_result[0])
                    except:
                        current_log = []
                    
                    current_log.append(log_entry)
                    cursor.execute(
                        'UPDATE research_history SET progress_log = ? WHERE id = ?',
                        (json.dumps(current_log), research_id)
                    )
                
                conn.commit()
                conn.close()
                
                # Emit a socket event
                try:
                    # Basic event data
                    event_data = {
                        'message': message,
                        'progress': adjusted_progress
                    }
                    
                    # Add log entry in full format for detailed logging on client
                    if metadata:
                        event_data['log_entry'] = log_entry
                    
                    # Send to all subscribers and broadcast channel
                    socketio.emit(f'research_progress_{research_id}', event_data)
                    
                    if research_id in socket_subscriptions:
                        for sid in socket_subscriptions[research_id]:
                            try:
                                socketio.emit(
                                    f'research_progress_{research_id}', 
                                    event_data, 
                                    room=sid
                                )
                            except Exception as err:
                                print(f"Error emitting to subscriber {sid}: {str(err)}")
                except Exception as e:
                    print(f"Socket emit error (non-critical): {str(e)}")
                    
        # FUNCTION TO CHECK TERMINATION DURING LONG-RUNNING OPERATIONS
        def check_termination():
            if research_id in termination_flags and termination_flags[research_id]:
                # Explicitly set the status to suspended in the database
                conn = sqlite3.connect(DB_PATH)
                cursor = conn.cursor()
                now = datetime.utcnow()
                completed_at = now.isoformat()
                
                cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
                result = cursor.fetchone()
                duration_seconds = calculate_duration(result[0]) if result and result[0] else None
                
                cursor.execute(
                    'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ? WHERE id = ?',
                    ('suspended', completed_at, duration_seconds, research_id)
                )
                conn.commit()
                conn.close()
                
                # Clean up resources
                cleanup_research_resources(research_id)
                
                # Raise exception to exit the process
                raise Exception("Research was terminated by user during long-running operation")
            return False  # Not terminated

        # Set the progress callback in the system
        system = AdvancedSearchSystem()
        system.set_progress_callback(progress_callback)
        
        # Run the search
        progress_callback("Starting research process", 5, {"phase": "init"})
        
        try:
            results = system.analyze_topic(query)
            if mode == 'quick':
                progress_callback("Search complete, preparing to generate summary...", 85, {"phase": "output_generation"})
            else:
                progress_callback("Search complete, generating output", 80, {"phase": "output_generation"})
        except Exception as search_error:
            # Better handling of specific search errors
            error_message = str(search_error)
            error_type = "unknown"
            
            # Extract error details for common issues
            if "status code: 503" in error_message:
                error_message = "Ollama AI service is unavailable (HTTP 503). Please check that Ollama is running properly on your system."
                error_type = "ollama_unavailable"
            elif "status code: 404" in error_message:
                error_message = "Ollama model not found (HTTP 404). Please check that you have pulled the required model."
                error_type = "model_not_found"
            elif "status code:" in error_message:
                # Extract the status code for other HTTP errors
                status_code = error_message.split("status code:")[1].strip()
                error_message = f"API request failed with status code {status_code}. Please check your configuration."
                error_type = "api_error"
            elif "connection" in error_message.lower():
                error_message = "Connection error. Please check that your LLM service (Ollama/API) is running and accessible."
                error_type = "connection_error"
            
            # Raise with improved error message
            raise Exception(f"{error_message} (Error type: {error_type})")
        
        # Generate output based on mode
        if mode == 'quick':
            # Quick Summary
            if results.get('findings'):

                raw_formatted_findings = results['formatted_findings']
                logger.info(f"Found formatted_findings of length: {len(str(raw_formatted_findings))}")
                
                try:
                    clean_markdown = raw_formatted_findings
                    # ADDED CODE: Convert debug output to clean markdown
                    #clean_markdown = convert_debug_to_markdown(raw_formatted_findings, query)
                    print(f"Successfully converted to clean markdown of length: {len(clean_markdown)}")
                    
                    # First send a progress update for generating the summary
                    progress_callback("Generating clean summary from research data...", 90, {"phase": "output_generation"})
                    
                    # Save as markdown file
                    output_dir = "research_outputs"
                    if not os.path.exists(output_dir):
                        os.makedirs(output_dir)
                        
                    safe_query = "".join(x for x in query if x.isalnum() or x in [" ", "-", "_"])[:50]
                    safe_query = safe_query.replace(" ", "_").lower()
                    report_path = os.path.join(output_dir, f"quick_summary_{safe_query}.md")
                    
                    # Send progress update for writing to file
                    progress_callback("Writing research report to file...", 95, {"phase": "report_complete"})
                    
                    print(f"Writing report to: {report_path}")
                    with open(report_path, "w", encoding="utf-8") as f:
                        f.write("# Quick Research Summary\n\n")
                        f.write(f"Query: {query}\n\n")
                        f.write(clean_markdown)  # Use clean markdown instead of raw findings
                        f.write("\n\n## Research Metrics\n")
                        f.write(f"- Search Iterations: {results['iterations']}\n")
                        f.write(f"- Generated at: {datetime.utcnow().isoformat()}\n")
                    
                    # Update database
                    metadata = {
                        'iterations': results['iterations'],
                        'generated_at': datetime.utcnow().isoformat()
                    }
                    
                    # Calculate duration in seconds - using UTC consistently
                    now = datetime.utcnow()
                    completed_at = now.isoformat()
                    
                    print(f"Updating database for research_id: {research_id}")
                    # Get the start time from the database
                    conn = sqlite3.connect(DB_PATH)
                    cursor = conn.cursor()
                    cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
                    result = cursor.fetchone()
                    
                    # Use the helper function for consistent duration calculation
                    duration_seconds = calculate_duration(result[0])
                    
                    # Update the record
                    cursor.execute(
                        'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, report_path = ?, metadata = ? WHERE id = ?',
                        ('completed', completed_at, duration_seconds, report_path, json.dumps(metadata), research_id)
                    )
                    conn.commit()
                    conn.close()
                    print(f"Database updated successfully for research_id: {research_id}")
                    
                    # Send the final completion message
                    progress_callback("Research completed successfully", 100, {"phase": "complete", "report_path": report_path})
                    
                    # Clean up resources
                    print(f"Cleaning up resources for research_id: {research_id}")
                    cleanup_research_resources(research_id)
                    print(f"Resources cleaned up for research_id: {research_id}")
                except Exception as inner_e:
                    print(f"Error during quick summary generation: {str(inner_e)}")
                    print(traceback.format_exc())
                    raise Exception(f"Error generating quick summary: {str(inner_e)}")
            else:
                raise Exception("No research findings were generated. Please try again.")
        else:
            # Full Report
            progress_callback("Generating detailed report...", 85, {"phase": "report_generation"})
            report_generator = IntegratedReportGenerator()
            final_report = report_generator.generate_report(results, query)
            progress_callback("Report generation complete", 95, {"phase": "report_complete"})
            
            # Save as markdown file
            output_dir = "research_outputs"
            if not os.path.exists(output_dir):
                os.makedirs(output_dir)
                
            safe_query = "".join(x for x in query if x.isalnum() or x in [" ", "-", "_"])[:50]
            safe_query = safe_query.replace(" ", "_").lower()
            report_path = os.path.join(output_dir, f"detailed_report_{safe_query}.md")
            
            with open(report_path, "w", encoding="utf-8") as f:
                f.write(final_report['content'])
            
            # Update database
            metadata = final_report['metadata']
            metadata['iterations'] = results['iterations']
            
            # Calculate duration in seconds - using UTC consistently
            now = datetime.utcnow()
            completed_at = now.isoformat()
            
            # Get the start time from the database
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
            result = cursor.fetchone()
            
            # Use the helper function for consistent duration calculation
            duration_seconds = calculate_duration(result[0])
            
            cursor.execute(
                'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, report_path = ?, metadata = ? WHERE id = ?',
                ('completed', completed_at, duration_seconds, report_path, json.dumps(metadata), research_id)
            )
            conn.commit()
            conn.close()
            
            progress_callback("Research completed successfully", 100, {"phase": "complete", "report_path": report_path})
            
            # Clean up - moved to a separate function for reuse
            cleanup_research_resources(research_id)
            
    except Exception as e:
        # Handle error
        error_message = f"Research failed: {str(e)}"
        print(f"Research error: {error_message}")
        try:
            # Check for common Ollama error patterns in the exception and provide more user-friendly errors
            user_friendly_error = str(e)
            error_context = {}
            
            if "Error type: ollama_unavailable" in user_friendly_error:
                user_friendly_error = "Ollama AI service is unavailable. Please check that Ollama is running properly on your system."
                error_context = {"solution": "Start Ollama with 'ollama serve' or check if it's installed correctly."}
            elif "Error type: model_not_found" in user_friendly_error:
                user_friendly_error = "Required Ollama model not found. Please pull the model first."
                error_context = {"solution": "Run 'ollama pull mistral' to download the required model."}
            elif "Error type: connection_error" in user_friendly_error:
                user_friendly_error = "Connection error with LLM service. Please check that your AI service is running."
                error_context = {"solution": "Ensure Ollama or your API service is running and accessible."}
            elif "Error type: api_error" in user_friendly_error:
                # Keep the original error message as it's already improved
                error_context = {"solution": "Check API configuration and credentials."}
            
            # Update metadata with more context about the error
            metadata = {
                "phase": "error", 
                "error": user_friendly_error
            }
            if error_context:
                metadata.update(error_context)
                
            progress_callback(user_friendly_error, None, metadata)
        
            conn = sqlite3.connect(DB_PATH)
            cursor = conn.cursor()
            
            # If termination was requested, mark as suspended instead of failed
            status = 'suspended' if (research_id in termination_flags and termination_flags[research_id]) else 'failed'
            message = "Research was terminated by user" if status == 'suspended' else user_friendly_error
            
            # Calculate duration up to termination point - using UTC consistently
            now = datetime.utcnow()
            completed_at = now.isoformat()
            
            # Get the start time from the database
            duration_seconds = None
            cursor.execute('SELECT created_at FROM research_history WHERE id = ?', (research_id,))
            result = cursor.fetchone()
            
            # Use the helper function for consistent duration calculation
            if result and result[0]:
                duration_seconds = calculate_duration(result[0])
            
            cursor.execute(
                'UPDATE research_history SET status = ?, completed_at = ?, duration_seconds = ?, metadata = ? WHERE id = ?',
                (status, completed_at, duration_seconds, json.dumps(metadata), research_id)
            )
            conn.commit()
            conn.close()
            
            try:
                socketio.emit(f'research_progress_{research_id}', {
                    'status': status,
                    'error': message
                })
                
                # Also notify specific subscribers
                if research_id in socket_subscriptions and socket_subscriptions[research_id]:
                    for sid in socket_subscriptions[research_id]:
                        try:
                            socketio.emit(
                                f'research_progress_{research_id}', 
                                {'status': status, 'error': message},
                                room=sid
                            )
                        except Exception as sub_err:
                            print(f"Error emitting to subscriber {sid}: {str(sub_err)}")
                            
            except Exception as socket_error:
                print(f"Failed to emit error via socket: {str(socket_error)}")
        except Exception as inner_e:
            print(f"Error in error handler: {str(inner_e)}")
        
        # Clean up resources - moved to a separate function for reuse
        cleanup_research_resources(research_id)

@research_bp.route('/api/research/<int:research_id>/terminate', methods=['POST'])
def terminate_research(research_id):
    """Terminate an in-progress research process"""
    
    # Check if the research exists and is in progress
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    cursor.execute('SELECT status FROM research_history WHERE id = ?', (research_id,))
    result = cursor.fetchone()
    
    if not result:
        conn.close()
        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
    
    status = result[0]
    
    # If it's not in progress, return an error
    if status != 'in_progress':
        conn.close()
        return jsonify({'status': 'error', 'message': 'Research is not in progress'}), 400
    
    # Check if it's in the active_research dict
    if research_id not in active_research:
        # Update the status in the database
        cursor.execute('UPDATE research_history SET status = ? WHERE id = ?', ('suspended', research_id))
        conn.commit()
        conn.close()
        return jsonify({'status': 'success', 'message': 'Research terminated'})
    
    # Set the termination flag
    termination_flags[research_id] = True
    
    # Log the termination request - using UTC timestamp
    timestamp = datetime.utcnow().isoformat()
    termination_message = "Research termination requested by user"
    current_progress = active_research[research_id]['progress']
    
    # Create log entry
    log_entry = {
        "time": timestamp,
        "message": termination_message,
        "progress": current_progress,
        "metadata": {"phase": "termination"}
    }
    
    # Add to in-memory log
    active_research[research_id]['log'].append(log_entry)
    
    # Add to database log
    add_log_to_db(
        research_id,
        termination_message,
        log_type='milestone',
        progress=current_progress,
        metadata={"phase": "termination"}
    )
    
    # Update the log in the database (old way for backward compatibility)
    cursor.execute('SELECT progress_log FROM research_history WHERE id = ?', (research_id,))
    log_result = cursor.fetchone()
    if log_result:
        try:
            current_log = json.loads(log_result[0])
        except:
            current_log = []
        current_log.append(log_entry)
        cursor.execute(
            'UPDATE research_history SET progress_log = ? WHERE id = ?',
            (json.dumps(current_log), research_id)
        )
    
    # IMMEDIATELY update the status to 'suspended' to avoid race conditions
    cursor.execute('UPDATE research_history SET status = ? WHERE id = ?', ('suspended', research_id))
    conn.commit()
    conn.close()
    
    # Emit a socket event for the termination request
    try:
        event_data = {
            'status': 'suspended',  # Changed from 'terminating' to 'suspended'
            'message': 'Research was suspended by user request'
        }
        
        socketio.emit(f'research_progress_{research_id}', event_data)
        
        if research_id in socket_subscriptions and socket_subscriptions[research_id]:
            for sid in socket_subscriptions[research_id]:
                try:
                    socketio.emit(
                        f'research_progress_{research_id}', 
                        event_data,
                        room=sid
                    )
                except Exception as err:
                    print(f"Error emitting to subscriber {sid}: {str(err)}")
                    
    except Exception as socket_error:
        print(f"Socket emit error (non-critical): {str(socket_error)}")
    
    return jsonify({'status': 'success', 'message': 'Research termination requested'})

@research_bp.route('/api/research/<int:research_id>/delete', methods=['DELETE'])
def delete_research(research_id):
    """Delete a research record"""
    conn = sqlite3.connect(DB_PATH)
    cursor = conn.cursor()
    
    # First check if the research exists and is not in progress
    cursor.execute('SELECT status, report_path FROM research_history WHERE id = ?', (research_id,))
    result = cursor.fetchone()
    
    if not result:
        conn.close()
        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
    
    status, report_path = result
    
    # Don't allow deleting research in progress
    if status == 'in_progress' and research_id in active_research:
        conn.close()
        return jsonify({
            'status': 'error', 
            'message': 'Cannot delete research that is in progress'
        }), 400
    
    # Delete report file if it exists
    if report_path and os.path.exists(report_path):
        try:
            os.remove(report_path)
        except Exception as e:
            print(f"Error removing report file: {str(e)}")
    
    # Delete the database record
    cursor.execute('DELETE FROM research_history WHERE id = ?', (research_id,))
    conn.commit()
    conn.close()
    
    return jsonify({'status': 'success'})
@research_bp.route('/settings', methods=['GET'])
def settings_page():
    """Main settings dashboard with links to specialized config pages"""
    return render_template('settings_dashboard.html')

@research_bp.route('/settings/main', methods=['GET'])
def main_config_page():
    """Edit main configuration with search parameters"""
    return render_template('main_config.html', main_file_path=MAIN_CONFIG_FILE)

@research_bp.route('/settings/llm', methods=['GET'])
def llm_config_page():
    """Edit LLM configuration using raw file editor"""
    return render_template('llm_config.html', llm_file_path=LLM_CONFIG_FILE)

@research_bp.route('/settings/collections', methods=['GET']) 
def collections_config_page():
    """Edit local collections configuration using raw file editor"""
    return render_template('collections_config.html', collections_file_path=LOCAL_COLLECTIONS_FILE)

@research_bp.route('/settings/api_keys', methods=['GET'])
def api_keys_config_page():
    """Edit API keys configuration"""
    # Get the secrets file path
    secrets_file = CONFIG_DIR / ".secrets.toml"
    
    return render_template('api_keys_config.html', secrets_file_path=secrets_file)
# Add to the imports section
from local_deep_research.config import SEARCH_ENGINES_FILE

# Add a new route for search engines configuration page
@research_bp.route('/settings/search_engines', methods=['GET'])
def search_engines_config_page():
    """Edit search engines configuration using raw file editor"""
    # Read the current config file
    raw_config = ""
    try:
        with open(SEARCH_ENGINES_FILE, 'r') as f:
            raw_config = f.read()
    except Exception as e:
        flash(f'Error reading search engines configuration: {str(e)}', 'error')
        raw_config = "# Error reading configuration file"
    
    # Get list of engine names for display
    engine_names = []
    try:
        from local_deep_research.web_search_engines.search_engines_config import SEARCH_ENGINES
        engine_names = list(SEARCH_ENGINES.keys())
        engine_names.sort()  # Alphabetical order
    except Exception as e:
        logger.error(f"Error getting engine names: {e}")
    
    return render_template('search_engines_config.html', 
                          search_engines_file_path=SEARCH_ENGINES_FILE,
                          raw_config=raw_config,
                          engine_names=engine_names)

# Add a route to save search engines configuration
@research_bp.route('/api/save_search_engines_config', methods=['POST'])
def save_search_engines_config():
    try:
        data = request.get_json()
        raw_config = data.get('raw_config', '')
        
        # Validate TOML syntax
        try:
            toml.loads(raw_config)
        except toml.TomlDecodeError as e:
            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
        
        # Ensure directory exists
        os.makedirs(os.path.dirname(SEARCH_ENGINES_FILE), exist_ok=True)
        
        # Create a backup first
        backup_path = f"{SEARCH_ENGINES_FILE}.bak"
        if os.path.exists(SEARCH_ENGINES_FILE):
            import shutil
            shutil.copy2(SEARCH_ENGINES_FILE, backup_path)
        
        # Write new config
        with open(SEARCH_ENGINES_FILE, 'w') as f:
            f.write(raw_config)
            
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})


# API endpoint to save raw LLM config
@research_bp.route('/api/save_llm_config', methods=['POST'])
def save_llm_config():
    try:
        data = request.get_json()
        raw_config = data.get('raw_config', '')
        
        # Validate Python syntax
        try:
            compile(raw_config, '<string>', 'exec')
        except SyntaxError as e:
            return jsonify({'success': False, 'error': f'Syntax error: {str(e)}'})
        
        # Ensure directory exists
        os.makedirs(os.path.dirname(LLM_CONFIG_FILE), exist_ok=True)
        
        # Create a backup first
        backup_path = f"{LLM_CONFIG_FILE}.bak"
        if os.path.exists(LLM_CONFIG_FILE):
            import shutil
            shutil.copy2(LLM_CONFIG_FILE, backup_path)
        
        # Write new config
        with open(LLM_CONFIG_FILE, 'w') as f:
            f.write(raw_config)
            
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})

# API endpoint to save raw collections config
@research_bp.route('/api/save_collections_config', methods=['POST'])
def save_collections_config():
    try:
        data = request.get_json()
        raw_config = data.get('raw_config', '')
        
        # Validate TOML syntax
        try:
            toml.loads(raw_config)
        except toml.TomlDecodeError as e:
            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
        
        # Ensure directory exists
        os.makedirs(os.path.dirname(LOCAL_COLLECTIONS_FILE), exist_ok=True)
        
        # Create a backup first
        backup_path = f"{LOCAL_COLLECTIONS_FILE}.bak"
        if os.path.exists(LOCAL_COLLECTIONS_FILE):
            import shutil
            shutil.copy2(LOCAL_COLLECTIONS_FILE, backup_path)
        
        # Write new config
        with open(LOCAL_COLLECTIONS_FILE, 'w') as f:
            f.write(raw_config)
        
        # Also trigger a reload in the collections system
        try:
            load_local_collections(reload=True)
        except Exception as reload_error:
            return jsonify({'success': True, 'warning': f'Config saved, but error reloading: {str(reload_error)}'})
            
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})

# API endpoint to save raw main config 
@research_bp.route('/api/save_main_config', methods=['POST'])
def save_raw_main_config():
    try:
        data = request.get_json()
        raw_config = data.get('raw_config', '')
        
        # Validate TOML syntax
        try:
            toml.loads(raw_config)
        except toml.TomlDecodeError as e:
            return jsonify({'success': False, 'error': f'TOML syntax error: {str(e)}'})
        
        # Ensure directory exists
        os.makedirs(os.path.dirname(MAIN_CONFIG_FILE), exist_ok=True)
        
        # Create a backup first
        backup_path = f"{MAIN_CONFIG_FILE}.bak"
        if os.path.exists(MAIN_CONFIG_FILE):
            import shutil
            shutil.copy2(MAIN_CONFIG_FILE, backup_path)
        
        # Write new config
        with open(MAIN_CONFIG_FILE, 'w') as f:
            f.write(raw_config)
            
        return jsonify({'success': True})
    except Exception as e:
        return jsonify({'success': False, 'error': str(e)})
@research_bp.route('/raw_config')
def get_raw_config():
    """Return the raw configuration file content"""
    try:
        # Determine which config file to load based on a query parameter
        config_type = request.args.get('type', 'main')
        
        if config_type == 'main':
            config_path = os.path.join(app.config['CONFIG_DIR'], 'config.toml')
            with open(config_path, 'r') as f:
                return f.read()
        elif config_type == 'llm':
            config_path = os.path.join(app.config['CONFIG_DIR'], 'llm_config.py')
            with open(config_path, 'r') as f:
                return f.read()
        elif config_type == 'collections':
            config_path = os.path.join(app.config['CONFIG_DIR'], 'collections.toml')
            with open(config_path, 'r') as f:
                return f.read()
        else:
            return "Unknown configuration type", 400
    except Exception as e:
        return str(e), 500
import os
import subprocess
import platform

@research_bp.route('/open_file_location', methods=['POST'])
def open_file_location():
    file_path = request.form.get('file_path')
    
    if not file_path:
        flash('No file path provided', 'error')
        return redirect(url_for('research.settings_page'))
    
    # Get the directory containing the file
    dir_path = os.path.dirname(os.path.abspath(file_path))
    
    # Open the directory in the file explorer
    try:
        if platform.system() == "Windows":
            subprocess.Popen(f'explorer "{dir_path}"')
        elif platform.system() == "Darwin":  # macOS
            subprocess.Popen(["open", dir_path])
        else:  # Linux
            subprocess.Popen(["xdg-open", dir_path])
        
        flash(f'Opening folder: {dir_path}', 'success')
    except Exception as e:
        flash(f'Error opening folder: {str(e)}', 'error')
    
    # Redirect back to the settings page
    if 'llm' in file_path:
        return redirect(url_for('research.llm_config_page'))
    elif 'collections' in file_path:
        return redirect(url_for('research.collections_config_page'))
    else:
        return redirect(url_for('research.main_config_page'))

@research_bp.route('/api/research/<int:research_id>/logs')
def get_research_logs(research_id):
    """Get logs for a specific research ID"""
    # First check if the research exists
    conn = sqlite3.connect(DB_PATH)
    conn.row_factory = sqlite3.Row
    cursor = conn.cursor()
    cursor.execute('SELECT id FROM research_history WHERE id = ?', (research_id,))
    result = cursor.fetchone()
    conn.close()
    
    if not result:
        return jsonify({'status': 'error', 'message': 'Research not found'}), 404
    
    # Retrieve logs from the database
    logs = get_logs_for_research(research_id)
    
    # Add any current logs from memory if this is an active research
    if research_id in active_research and active_research[research_id].get('log'):
        # Use the logs from memory temporarily until they're saved to the database
        memory_logs = active_research[research_id]['log']
        
        # Filter out logs that are already in the database
        # We'll compare timestamps to avoid duplicates
        db_timestamps = {log['time'] for log in logs}
        unique_memory_logs = [log for log in memory_logs if log['time'] not in db_timestamps]
        
        # Add unique memory logs to our return list
        logs.extend(unique_memory_logs)
        
        # Sort logs by timestamp
        logs.sort(key=lambda x: x['time'])
    
    return jsonify({
        'status': 'success',
        'logs': logs
    })



# Register the blueprint
app.register_blueprint(research_bp)

# Also add the static route at the app level for compatibility
@app.route('/static/<path:path>')
def app_serve_static(path):
    return send_from_directory(app.static_folder, path)

# Add favicon route to prevent 404 errors
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(app.static_folder, 'favicon.ico', mimetype='image/x-icon')

# Add this function to app.py
def convert_debug_to_markdown(raw_text, query):
    """
    Convert the debug-formatted text to clean markdown.
    
    Args:
        raw_text: The raw formatted findings with debug symbols
        query: Original research query
    
    Returns:
        Clean markdown formatted text
    """
    try:
        print(f"Starting markdown conversion for query: {query}")
        print(f"Raw text type: {type(raw_text)}")
        
        # Handle None or empty input
        if not raw_text:
            print("WARNING: raw_text is empty or None")
            return f"No detailed findings available for '{query}'."
            
        # If there's a "DETAILED FINDINGS:" section, extract everything after it
        if "DETAILED FINDINGS:" in raw_text:
            print("Found DETAILED FINDINGS section")
            detailed_index = raw_text.index("DETAILED FINDINGS:")
            content = raw_text[detailed_index + len("DETAILED FINDINGS:"):].strip()
        else:
            print("No DETAILED FINDINGS section found, using full text")
            content = raw_text
        
        # Remove divider lines with === symbols
        lines_before = len(content.split("\n"))
        content = "\n".join([line for line in content.split("\n") 
                            if not line.strip().startswith("===") and not line.strip() == "="*80])
        lines_after = len(content.split("\n"))
        print(f"Removed {lines_before - lines_after} divider lines")
        

        
        # Remove SEARCH QUESTIONS BY ITERATION section
        if "SEARCH QUESTIONS BY ITERATION:" in content:
            print("Found SEARCH QUESTIONS BY ITERATION section")
            search_index = content.index("SEARCH QUESTIONS BY ITERATION:")
            next_major_section = -1
            for marker in ["DETAILED FINDINGS:", "COMPLETE RESEARCH:"]:
                if marker in content[search_index:]:
                    marker_pos = content.index(marker, search_index)
                    if next_major_section == -1 or marker_pos < next_major_section:
                        next_major_section = marker_pos
            
            if next_major_section != -1:
                print(f"Removing section from index {search_index} to {next_major_section}")
                content = content[:search_index] + content[next_major_section:]
            else:
                # If no later section, just remove everything from SEARCH QUESTIONS onwards
                print(f"Removing everything after index {search_index}")
                content = content[:search_index].strip()
        
        print(f"Final markdown length: {len(content.strip())}")
        return content.strip()
    except Exception as e:
        print(f"Error in convert_debug_to_markdown: {str(e)}")
        print(traceback.format_exc())
        # Return a basic message with the original query as fallback
        return f"# Research on {query}\n\nThere was an error formatting the research results."

def main():
    """
    Entry point for the web application when run as a command.
    This function is needed for the package's entry point to work properly.
    """
    # Import settings here to avoid circular imports
    from local_deep_research.config import settings

    # Get web server settings with defaults
    port = settings.web.port
    host = settings.web.host
    debug = settings.web.debug

    # Check for OpenAI availability but don't import it unless necessary
    try:
        import os
        api_key = os.environ.get("OPENAI_API_KEY")
        if api_key:
            try:
                # Only try to import if we have an API key
                import openai
                openai.api_key = api_key
                OPENAI_AVAILABLE = True
                print("OpenAI integration is available")
            except ImportError:
                print("OpenAI package not installed, integration disabled")
        else:
            print("OPENAI_API_KEY not found in environment variables, OpenAI integration disabled")
    except Exception as e:
        print(f"Error checking OpenAI availability: {e}")
        
    socketio.run(app, debug=debug, host=host, port=port, allow_unsafe_werkzeug=True)
    
if __name__ == '__main__':
    main()