from datetime import datetime
from scrapy import Spider, signals
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.settings import BaseSettings

import logging
import pickle
import pymongo

logger = logging.getLogger(__name__)


class TraceErrorBase:
    def __init__(self, settings: BaseSettings):
        self.url = settings.get('ERROR_MONGO_URL') or settings.get('MONGO_URL')
        assert self.url, 'ERROR_MONGO_URL or MONGO_URL is required'

        self.db_name = settings.get('ERROR_MONGO_DATABASE', 'scraping')
        self.col_name = settings.get('ERROR_MONGO_COLLECTION', 'errors')

        self._client = None
        self._col = None

    def open_spider(self, spider: Spider):
        logger.debug("Connecting to MongoDB...")
        self._client = pymongo.MongoClient(self.url)
        self._col = self._client[self.db_name][self.col_name]

    def close_spider(self, spider: Spider):
        logger.debug(f"Closing MongoDB connection...")
        self._client.close()


class TraceErrorSpiderMiddleware(TraceErrorBase):
    @classmethod
    def from_crawler(cls, crawler: Crawler):
        s = cls(crawler.settings)
        crawler.signals.connect(s.close_spider, signals.spider_closed)
        crawler.signals.connect(s.open_spider, signals.spider_opened)
        return s

    def process_spider_exception(self, response: Response, exception, spider: Spider):
        response_str = str(response)
        exception_str = str(exception)

        request = response.request
        logger.debug("Storing error response %s %s for %s" % (request.method, response.url, exception_str))

        details = {
            'headers': dict(response.headers),
            'body': response.text,
        }
        
        item = {
            'type': 'response',
            'created_at': datetime.now(),
            'spider': spider.name,
            'message': exception_str,
            'method': request.method,
            'url': request.url,
            'status': response.status,
            'response': response_str,
            'pickle': pickle.dumps(details, protocol=4)
        }
        
        self._col.insert_one(item)


class TraceErrorDownloaderMiddleware(TraceErrorBase):
    @classmethod
    def from_crawler(cls, crawler: Crawler):
        s = cls(crawler.settings)
        crawler.signals.connect(s.close_spider, signals.spider_closed)
        crawler.signals.connect(s.open_spider, signals.spider_opened)
        return s
    
    def process_exception(self, request, exception, spider):
        exception_str = str(exception)
    
        logger.debug("Storing error request %s %s for %s" % (request.method, request.url, exception_str))

        data = {
            'headers': dict(request.headers),
        }
        
        item = {
            'type': 'request',
            'created_at': datetime.now(),
            'spider': spider.name,
            'message': exception_str,
            'method': request.method,
            'url': request.url,
            'pickle': pickle.dumps(data, protocol=4)
        }
    
        self._col.insert_one(item)
