import urllib.request
from bs4 import BeautifulSoup
import re
import requests

from Spitzer.exploiters.web.dirbuster import bust
from Spitzer.result import result
from Spitzer.config.config import get_data

category = ['heavy']

def exploit(url, nmap):
    pages = bust(url, nmap)
    spider(url, pages)

def spider(base_url, urls):
    print('[*] spidering!')
    if not base_url.endswith('/'):
        base_url = base_url + '/'

    if urls is None:
        urls = []

    if base_url not in urls:
        urls.append(base_url)


    newpages = urls
    pages = []

    while newpages != []:
        for url in newpages:
            print('[*] Found: '+str(len(pages)) + '\tTo scan:' + str(len(newpages)), end='\r')
            if not is_url(url):
                newpages.remove(url)
                continue
            elif is_link(url):
                url = base_url + url

            try:
                print(url)
                page = requests.get(url).text
            except urllib.error.HTTPError as e:
                if e in ('200','204','301','302','307','401','403'):
                    print('error')
                    pages.append(url)
                    newpages.remove(url)
                    break
                else:
                    newpages.remove(url)
                continue
            except (UnicodeDecodeError, AttributeError, OSError):
                pass

            if '<!doctype' not in page:
                print('not a page')
                newpages = get_js(page, base_url, newpages, pages)
                pages.append(url)
                newpages.remove(url)
                continue

            soup = BeautifulSoup(page, features='lxml')

            links = soup.find_all('a')
            newpages = filter_a(links, base_url, newpages, pages)

            links = soup.find_all('script')
            newpages = filter_js(links, base_url, newpages, pages)
            
            pages.append(url)
            newpages.remove(url)

    print('\n')
    result.add_pages(base_url, pages)

def filter_a(links, base_url, newpages, pages):
    for tag in links:
        link = tag.get('href',None)
        if link is None:
            continue
        link = link.split('#')[0]
        link = link.split('?')[0]
        if is_link(link):
            link = base_url[:-1] + link
        if not compare(link, base_url) or link in newpages or link in pages:
            continue
        print(link)
        newpages.append(link)
    return newpages

def filter_js(links, base_url, newpages, pages):
    for tag in links:
        js = tag.get('src',None)#if it has no src, use the body
        if js is None:
            js = tag.contents
            if js is None:
                continue

        newpages = get_js(js, base_url, newpages, pages)

    return newpages

def get_js(js, base_url, newpages, pages):
    #find every string between single and double qoutes:
    double = re.findall(r'"([^"]*)"', str(js))
    single = re.findall(r"'([^']*)'", str(js))

    strings = double + single
    
    for string in strings:
        if is_link(string):
            #it is a link!
            u = ''
            if string.startswith('/'):
                u = base_url + string
            else:
                u = base_url + '/' + string
            if u not in newpages or u not in pages:
                print(u)
                newpages.append(u)

    return newpages


def compare(link, base_url):
    b = base_url.replace('https://', '').replace('http://', '').replace('www.', '')
    l = link.replace('https://', '').replace('http://', '').replace('www.', '')
    return b in l
    
def is_link(link):
    ex = get_data('extensions')
    extentions = []
    for _, value in ex.items():
        extentions += value.split(' ')

    extentions = tuple(extentions)

    if link.startswith('/') or link.endswith(extentions):
        return True

    if link.startswith(('mailto', 'callto')):
        return False

    return False

def is_url(link):
    ex = get_data('extensions')
    extentions = []
    for _, value in ex.items():
        extentions += value.split(' ')

    extentions = tuple(extentions)

    if link.startswith('http') and link.endswith(extentions):
        return True
    if link.startswith(('mailto', 'callto')):
        return False
    return False