import urllib.robotparser as u
from  urllib.parse import urlparse
import requests as r
from bs4 import BeautifulSoup as bs
class CrawlerBot:
    def __init__(self,name:str)->None:
        self.name=name
        self.robot_parser=u.RobotFileParser()
    def is_url_crawlable(self,url):
        robot_url= urlparse(url)._replace(path="/robots.txt",
params='',
            query='', fragment='').geturl()
        allowed_to_crawl=False
        self.robot_parser.set_url(robot_url)
        #print(self.robot_parser.read())
        if self.robot_parser.can_fetch("*",url):
            print("url is allowed to crawl")
            allowed_to_crawl=True
        else:
            print("url is not allowed to crawl") 
        #print(robot_url)
        return robot_url,allowed_to_crawl
    def get_content_from_urls(self,urls:list)->list:
        return_data_list=[]
        response= [r.get(i) for i in urls]
        #print(response[0].encoding,
        #[i for i in response[0].raw.read_chunked()],[dir(response[0].raw)],
        #dir(response[0]))
        for i in response:
            if i.status_code==200:
                soup=bs(i.content,"html.parser")
                return_data_list.append(soup.prettify())
            # print(soup.find("body"))
        return return_data_list
bot=CrawlerBot("JJSRBot")
#print(bot.is_url_crawlable("https://docs.python.org/3/library/urllib.parse.html#module-urllib.parse"))
#bot.is_url_crawlable("https://shree-hari-satta.serv00.net")


