import copy

from page_scraper.entities.models import PageContext
from page_scraper.meta.utils import get_meta_by_name


class RobotScraper:
    def run(self,page:PageContext) -> PageContext:
        soup_copy = copy.copy(page.soup)
        robots = get_meta_by_name(soup_copy, 'robots')[0].get('content','')

        page.is_index = False if 'noindex' in robots else True
        page.is_follow = False if 'nofollow' in robots else True

        return page