from bs4 import BeautifulSoup
import json

from page_scraper.core.node_extractor import extract_nodes
from page_scraper.core.node_utils import build_entity_index
from page_scraper.core.utils import is_root_url


class PageParser:
    def parse(self, response):
        if response is None:
            return None

        soup = BeautifulSoup(response.content, "html.parser")
        is_domain = is_root_url(response.url)
        nodes = self.extract_jsonld(soup)
        return {
            "url": response.url,
            "status_code": response.status_code,
            "headers": response.headers,
            "nodes": nodes,
            "index_nodes": build_entity_index(nodes),
            "is_domain": is_domain,
        }

    def extract_jsonld(self, soup:BeautifulSoup):
        blocks = []
        for block in soup.find_all("script", type="application/ld+json"):
            try:
                blocks.append(json.loads(block.string))
            except Exception as e:
                pass

        nodes = extract_nodes(blocks)

        return nodes