import urllib.robotparser as u
from  urllib.parse import urlparse,urlsplit
import requests as r
from bs4 import BeautifulSoup as bs
class CrawlerBot:
    def __init__(self,name:str)->None:
        self.name=name
        self.robot_parser=u.RobotFileParser()
        self.headers = {
      'User-Agent': 'JJSRCrawlerBot/1.0 all.dart603@passinbox for inqueries'
        }
    def get_root_path(self,url):
        url1= urlsplit(url)
        root_url= f"{url1[0]}://{url1[1]}"
        return root_url

    def is_url_crawlable(self,url):
        url1= self.get_root_path(url)
        robot_url= f"{url1}/robots.txt"
        allowed_to_crawl=False
        try:
            with r.get(robot_url,headers=self.headers) as response:
                if response.status_code==200:
                    self.robot_parser.parse(response.text.splitlines())
                    self.robot_parser.modified()
                    #print(self.robot_parser.read())
                    if self.robot_parser.can_fetch("*",url):
                        #print("url is allowed to crawl")
                        allowed_to_crawl=True
                else:
                    allowed_to_crawl=True
        except BaseException as e:
            return str(e)
        return robot_url,allowed_to_crawl
    def get_content_from_urls(self,urls:list)->list:
        return_data_list=[]
        crawlable=self.is_url_crawlable
        response= [r.get(i,headers=self.headers) for i in urls if crawlable(i)[1]]
        #print(response[0].encoding,
        #[i for i in response[0].raw.read_chunked()],[dir(response[0].raw)],
        #dir(response[0]))
        for i in response:
            if i.status_code==200:
                soup=bs(i.content,"html.parser")
                url_data=[]
                url_data.append(soup.title.string if not soup.title==None else " ")
                url_data.append(soup.h1.string if not soup.h1 ==None else " ") 
                return_data_list.append(url_data)
            # print(soup.find("body"))
        return return_data_list
bot=CrawlerBot("JJSRBot")
urls=["https://shree-hari-satta.serv00.net/",
"https://comdotharman0.github.io/",
"https://satta-king-fast.com/"]
#print(bot.get_content_from_urls([urls[2]])

