import csv
import json
from pathlib import Path
from datetime import datetime
from fsm_crawl.blocking.labeler import Labeler
from fsm_crawl.monitoring.callback_pipeline import GenericCallbackPipeline

class RequestLoggingPipeline(GenericCallbackPipeline):
    def __init__(
        self,
        labeler: Labeler,
        output_dir: str = "crawl_logs",
        filename_prefix: str = "requests"
    ):
        self.labeler = labeler
        self.output_dir = Path(output_dir)
        self.output_dir.mkdir(parents=True, exist_ok=True)

        self.filename_prefix = filename_prefix
        self.split_index = 0
        self.file = None
        self.writer = None

        self._open_new_file()

    # -------------------------
    # File handling
    # -------------------------
    def _open_new_file(self):
        if self.file:
            self.file.close()

        ts = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
        filename = f"{self.filename_prefix}_{self.split_index}_{ts}.csv"
        path = self.output_dir / filename

        self.file = open(path, "w", newline="", encoding="utf-8")
        self.writer = csv.DictWriter(
            self.file,
            fieldnames=[
                "timestamp",
                "method",
                "resource_type",
                "url",
                "frame_url",
                "headers",
                "label",
            ],
        )
        self.writer.writeheader()
        
    def on_response(self, response):
        pass

    async def on_request(self, route, request):
        try:
            label = self.labeler.label(request.url)
            headers = request.headers

            row = {
                "timestamp": datetime.utcnow().isoformat(),
                "method": request.method,
                "resource_type": request.resource_type,
                "url": request.url,
                "frame_url": request.frame.url if request.frame else None,
                "headers": json.dumps(headers, ensure_ascii=False),
                "label": label,
            }

            self.writer.writerow(row)

        except Exception as e:
            print(f"[Pipeline] Error logging request: {e}")

        finally:
            await route.continue_()

    async def on_iteration_split(self):
        self.split_index += 1
        self._open_new_file()

    async def on_crawl_end(self):
        if self.file:
            self.file.close()
