import os
import unittest
import logging
import sys
import time
from requests import api
from requests.exceptions import HTTPError, RequestException

from .api import WaterCrawlAPIClient


# Set up logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
    stream=sys.stdout
)
logger = logging.getLogger('WaterCrawlTest')


class TestWaterCrawlAPI(unittest.TestCase):
    def setUp(self):
        self.api = WaterCrawlAPIClient(
            api_key=os.environ['WATERCRAWL_API_KEY'],
        )
        
    def log_request_response(self, response, method_name):
        """Log detailed request and response information for debugging"""
        try:
            logger.info(f"API {method_name} Request URL: {response.request.url}")
            logger.info(f"API {method_name} Request Method: {response.request.method}")
            # Don't log the full headers as they contain API keys
            logger.info(f"API {method_name} Response Status: {response.status_code}")
            
            # Safely try to log response content
            try:
                if 'application/json' in response.headers.get('Content-Type', ''):
                    logger.info(f"API {method_name} Response: {response.json()}")
                else:
                    content_type = response.headers.get('Content-Type', 'unknown')
                    content_length = len(response.content) if response.content else 0
                    logger.info(f"API {method_name} Response: [Content-Type: {content_type}, Length: {content_length}]")
            except Exception as e:
                logger.info(f"API {method_name} Response: Could not parse response - {str(e)}")
        except Exception as e:
            logger.error(f"Error logging request/response details: {str(e)}")
    
    def handle_api_error(self, e, method_name):
        """Standardized error handling for API calls"""
        logger.error(f"{method_name} API Error: {str(e)}")
        logger.error(f"Error type: {type(e).__name__}")
        
        if isinstance(e, HTTPError) and hasattr(e, 'response'):
            logger.error(f"Status code: {e.response.status_code}")
            logger.error(f"Response text: {e.response.text}")
            logger.error(f"Request URL: {e.response.request.url}")
            logger.error(f"Request method: {e.response.request.method}")
        
        return f"API call to {method_name} failed: {str(e)}"
    
    def retry_api_call(self, func, *args, max_retries=3, **kwargs):
        """Retry an API call with exponential backoff"""
        last_exception = None
        for attempt in range(max_retries):
            try:
                if attempt > 0:
                    logger.info(f"Retry attempt {attempt+1}/{max_retries}")
                return func(*args, **kwargs)
            except (HTTPError, RequestException) as e:
                last_exception = e
                # Don't retry if it's a client error (4xx) except for 429 (too many requests)
                if isinstance(e, HTTPError) and hasattr(e, 'response'):
                    if 400 <= e.response.status_code < 500 and e.response.status_code != 429:
                        logger.error(f"Client error {e.response.status_code}, not retrying")
                        break
                
                # Exponential backoff
                wait_time = (2 ** attempt) * 0.5  # 0.5, 1, 2 seconds
                logger.warning(f"API call failed: {str(e)}. Waiting {wait_time}s before retry.")
                time.sleep(wait_time)
        
        # If we got here, all retries failed
        if last_exception:
            raise last_exception
        raise Exception("All retries failed for unknown reason")

    def test_get_crawl_requests_list(self):
        try:
            response = self.retry_api_call(self.api.get_crawl_requests_list)
            logger.info(f"Crawl Requests List: {response}")
            self.assertIsInstance(response['results'], list)
        except Exception as e:
            error_msg = self.handle_api_error(e, "get_crawl_requests_list")
            self.fail(error_msg)

    def test_get_crawl_request(self):
        try:
            items = self.retry_api_call(self.api.get_crawl_requests_list)
            response = self.retry_api_call(self.api.get_crawl_request, items['results'][0]['uuid'])
            logger.info(f"Crawl Request Details: {response}")
            self.assertIsInstance(response, dict)
        except Exception as e:
            error_msg = self.handle_api_error(e, "get_crawl_request")
            self.fail(error_msg)

    def test_create_crawl_request(self):
        try:
            response = self.retry_api_call(self.api.create_crawl_request, url='https://watercrawl.dev')
            logger.info(f"Created Crawl Request: {response}")
            self.assertIsInstance(response, dict)
        except Exception as e:
            error_msg = self.handle_api_error(e, "create_crawl_request")
            self.fail(error_msg)

    def test_stop_crawl_request(self):
        try:
            result = self.retry_api_call(self.api.create_crawl_request, url='https://watercrawl.dev')
            response = self.retry_api_call(self.api.stop_crawl_request, result['uuid'])
            logger.info(f"Stopped Crawl Request Result: {response}")
            self.assertIsNone(response)
        except Exception as e:
            error_msg = self.handle_api_error(e, "stop_crawl_request")
            self.fail(error_msg)

    def test_download_crawl_request(self):
        try:
            result = self.retry_api_call(self.api.get_crawl_requests_list)
            response = self.retry_api_call(self.api.download_crawl_request, result['results'][0]['uuid'])
            logger.info(f"Download Crawl Request Size: {len(response)} bytes")
            self.assertIsInstance(response, bytes)
        except Exception as e:
            error_msg = self.handle_api_error(e, "download_crawl_request")
            self.fail(error_msg)

    def test_monitor_crawl_request(self):
        try:
            result = self.retry_api_call(self.api.create_crawl_request, url='https://watercrawl.dev')
            logger.info(f"Created Request for Monitoring: {result}")
            count = 0
            for item in self.retry_api_call(self.api.monitor_crawl_request, result['uuid'], download=False):
                logger.info(f"Monitor Event {count}: {item}")
                self.assertIsInstance(item, dict)
                count += 1
                # Limit the number of events to avoid long test runs
                if count >= 3:
                    break
        except Exception as e:
            error_msg = self.handle_api_error(e, "monitor_crawl_request")
            self.fail(error_msg)

    def test_get_crawl_request_results(self):
        try:
            result = self.retry_api_call(self.api.get_crawl_requests_list)
            response = self.retry_api_call(self.api.get_crawl_request_results, result['results'][-1]['uuid'])
            logger.info(f"Crawl Request Results: {response}")
            self.assertIsInstance(response['results'], list)
        except Exception as e:
            error_msg = self.handle_api_error(e, "get_crawl_request_results")
            self.fail(error_msg)

    def test_download_result(self):
        try:
            result_crawl = self.retry_api_call(self.api.get_crawl_requests_list)
            index = 0
            while True:
                result = self.retry_api_call(self.api.get_crawl_request_results, result_crawl['results'][index]['uuid'])
                if len(result['results']) > 0:
                    break
                index += 1
                if index >= len(result_crawl['results']):
                    self.skipTest("No crawl requests with results found")
                    return

            result = self.retry_api_call(self.api.get_crawl_request_results, result_crawl['results'][index]['uuid'])
            logger.info(f"Result to download: {result['results'][0]}")
            response = self.retry_api_call(self.api.download_result, result['results'][0])
            logger.info(f"Downloaded Result: {response}")
            self.assertIsInstance(response, dict)
        except Exception as e:
            error_msg = self.handle_api_error(e, "download_result")
            self.fail(error_msg)

    def test_scrape_url(self):
        try:
            response = self.retry_api_call(self.api.scrape_url, url='https://watercrawl.dev')
            logger.info(f"Scrape URL Response: {response}")
            self.assertIsInstance(response, dict)
        except Exception as e:
            error_msg = self.handle_api_error(e, "scrape_url")
            self.fail(error_msg)

    def test_download_sitemap(self):
        # Get a completed crawl request first
        try:
            crawl_requests = self.retry_api_call(self.api.get_crawl_requests_list)
            logger.info(f"Crawl Requests: {len(crawl_requests['results'])} found")
            
            has_sitemap = False
            # Find a crawl request with a sitemap
            for request in crawl_requests['results']:
                try:
                    crawl_request = self.retry_api_call(self.api.get_crawl_request, request['uuid'])
                    logger.info(f"Checking request {request['uuid']} for sitemap")
                    if 'sitemap' in crawl_request and crawl_request['sitemap'] is not None:
                        logger.info(f"Found crawl request with sitemap: {crawl_request['uuid']}, URL: {crawl_request['sitemap']}")
                        try:
                            result = self.retry_api_call(self.api.download_sitemap, crawl_request)
                            logger.info(f"Sitemap content type: {type(result)}")
                            logger.info(f"Sitemap content sample: {result[:2] if isinstance(result, list) else result}")
                            self.assertIsInstance(result, list)
                            if len(result) > 0:
                                logger.info(f"First sitemap entry: {result[0]}")
                                self.assertIn('url', result[0])
                                self.assertIn('title', result[0])
                            has_sitemap = True
                            break
                        except Exception as e:
                            logger.error(f"Error downloading sitemap for {request['uuid']}: {str(e)}")
                            if isinstance(e, HTTPError) and hasattr(e, 'response'):
                                logger.error(f"Status code: {e.response.status_code}")
                                logger.error(f"Response text: {e.response.text}")
                except Exception as e:
                    logger.error(f"Error checking request {request['uuid']}: {str(e)}")
                    continue
                    
            if not has_sitemap:
                self.skipTest("No crawl requests with valid sitemap found")
        except Exception as e:
            error_msg = self.handle_api_error(e, "download_sitemap")
            self.fail(error_msg)

    # Skip sitemap graph and markdown tests as they appear to return 404s
    def test_download_sitemap_graph(self):
        # Get a completed crawl request first
        crawl_requests = self.api.get_crawl_requests_list()
        
        # Find a crawl request with a sitemap
        for request in crawl_requests['results']:
            crawl_request = self.api.get_crawl_request(request['uuid'])
            if crawl_request['status'] == 'finished' and crawl_request['sitemap'] is not None:
                logger.info(f"Found crawl request for graph: {crawl_request['uuid']}")
                result = self.api.download_sitemap_graph(crawl_request)
                logger.info(f"Sitemap graph type: {type(result)}")
                self.assertIsNotNone(result)
                return
                
        self.skipTest("No crawl requests with sitemap found")

    def test_download_sitemap_markdown(self):
        # Get a completed crawl request first
        crawl_requests = self.api.get_crawl_requests_list()
        
        # Find a crawl request with a sitemap
        for request in crawl_requests['results']:
            crawl_request = self.api.get_crawl_request(request['uuid'])
            if crawl_request['status'] == 'finished' and crawl_request['sitemap'] is not None:
                logger.info(f"Found crawl request for markdown: {crawl_request['uuid']}")
                result = self.api.download_sitemap_markdown(crawl_request)
                logger.info(f"Sitemap markdown type: {type(result)}")
                self.assertIsNotNone(result)
                return
                
        self.skipTest("No crawl requests with sitemap found")

    def test_get_crawl_request_for_sitemap_errors(self):
        # Should raise ValueError if sitemap is missing
        with self.assertRaises(ValueError):
            self.api._WaterCrawlAPIClient__get_crawl_request_for_sitemap({})
            
    def test_create_search_request(self):
        # Test with sync=False only since sync=True seems to have issues
        try:
            logger.info("Testing create_search_request in async mode")
            async_response = self.retry_api_call(self.api.create_search_request, query='python programming', sync=False)
            logger.info(f"Async Search Request Created: {async_response}")
            self.assertIsInstance(async_response, dict)
            self.assertIn('uuid', async_response)
        except Exception as e:
            error_msg = self.handle_api_error(e, "create_search_request")
            self.fail(error_msg)
        
        # Test sync mode with a small result limit to be faster
        logger.info("Testing create_search_request in sync mode")
        sync_response = self.api.create_search_request(
            query='python library', 
            result_limit=2,
            sync=True
        )
        logger.info(f"Sync Search Request Result: {sync_response}")
        self.assertIsInstance(sync_response, dict)

    def test_monitor_search_request(self):
        # Create a search request to monitor
        try:
            logger.info("Creating search request for monitoring")
            search_request = self.retry_api_call(self.api.create_search_request, query='python tutorial', result_limit=2, sync=False)
            
            logger.info(f"Monitoring search request: {search_request['uuid']}")
            count = 0
            for event in self.retry_api_call(self.api.monitor_search_request, search_request['uuid'], download=False):
                logger.info(f"Monitor event {count}: {event}")
                self.assertIsInstance(event, dict)
                count += 1
                # Limit to avoid long test runs
                if count >= 3:
                    break
        except Exception as e:
            error_msg = self.handle_api_error(e, "monitor_search_request")
            self.fail(error_msg)
    
    def test_get_search_request(self):
        # Create a request first
        try:
            search_request = self.retry_api_call(self.api.create_search_request, query='python book', result_limit=2, sync=False)
            logger.info(f"Created search request: {search_request['uuid']}")
            
            # Test get_search_request with download=False (default)
            response = self.retry_api_call(self.api.get_search_request, search_request['uuid'])
            logger.info(f"Retrieved search request (download=False): {response}")
            self.assertIsInstance(response, dict)
            self.assertEqual(response['uuid'], search_request['uuid'])
            
            for event in self.api.monitor_search_request(search_request['uuid'], download=False):
                if event['type'] == 'state':
                    if event['data']['status'] == 'finished':
                        break

            download_response = self.api.get_search_request(search_request['uuid'], download=True)
                
            # If result exists, ensure it's not a string when download=True
            if 'result' in download_response:
                self.assertIsInstance(download_response['result'], (list, dict))
                self.assertNotIsInstance(download_response['result'], str)
            else:
                logger.warning("Response doesn't contain 'result' key for testing")

        except Exception as e:
            error_msg = self.handle_api_error(e, "get_search_request")
            self.fail(error_msg)
    
    def test_stop_search_request(self):
        # Create a request to stop
        search_request = self.api.create_search_request(
            query='python course', 
            result_limit=3,
            sync=False
        )
        logger.info(f"Created search request to stop: {search_request['uuid']}")
        
        # Wait for the request to start
        time.sleep(1)
        
        # Stop the request
        response = self.api.stop_search_request(search_request['uuid'])
        logger.info(f"Stop search request response: {response}")
        self.assertIsNone(response)
        
        # Verify it was stopped
        status = self.api.get_search_request(search_request['uuid'])
        logger.info(f"Status after stopping: {status}")
        self.assertIn('status', status)


if __name__ == '__main__':
    unittest.main()
