# Web scraper looking for data on the web for a certain paper


from selenium import webdriver
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
from bs4 import BeautifulSoup
from EcoOpen.utils.keywords import keywords
import os
import requests
from pathlib import Path
from itertools import chain


from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.firefox.service import Service as FirefoxService
from selenium.webdriver.edge.service import Service as EdgeService
from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.firefox import GeckoDriverManager
from webdriver_manager.microsoft import EdgeChromiumDriverManager

def get_webdriver():
    try:
        # Try Chrome
        chrome_service = ChromeService(ChromeDriverManager().install())
        return webdriver.Chrome(service=chrome_service)
    except Exception as e:
        print(f"Chrome WebDriver not found or failed to start: {e}")
    
    try:
        # Try Firefox
        firefox_service = FirefoxService(GeckoDriverManager().install())
        return webdriver.Firefox(service=firefox_service)
    except Exception as e:
        print(f"Firefox WebDriver not found or failed to start: {e}")
    
    try:
        # Try Edge
        edge_service = EdgeService(EdgeChromiumDriverManager().install())
        return webdriver.Edge(service=edge_service)
    except Exception as e:
        print(f"Edge WebDriver not found or failed to start: {e}")

    return None

# Usage example
# driver = get_webdriver()
# if driver:
#     driver.get("https://www.example.com")
#     # Perform actions with the driver
#     driver.quit()
# else:
#     print("No available WebDriver found.")


urls = [
    'https://link.springer.com/article/10.1007/s10886-017-0919-8',
    "https://link.springer.com/article/10.1007/s10886-018-0942-4",
    "https://doi.org/10.1073/pnas.1211733109",
    "http://dx.doi.org/10.3955/046.091.0105",
    "https://doi.org/10.1016/j.scitotenv.2013.10.121",
    "https://doi.org/10.1111/nph.14333"
    ]

filetypes = [
    "csv", "xlsx", "xls", "txt", "pdf", "zip",
    "tar.gz", "tar", "gz", "json", "docx",
    "doc", "ods", "odt", "pptx", "ppt", "png", "jpg", "md"]

exclude = []
for i in filetypes:
    exclude.append(i.lower()+".")

def find_data_web(doi):
    url = "https://doi.org/" + doi
    driver = get_webdriver()
    driver.maximize_window()
    driver.get(url)
    wait = WebDriverWait(driver, 10)
    real_url = driver.current_url
    print(real_url)
    domain = real_url.split('/')[2]
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    driver.quit()
    supplementary_files = []
    closed_article_snippets = [
        "buy article",
        "access the full article",
        "purchase pdf",
    ]
    # check if the article is closed
    if any(snippet in str(soup).lower() for snippet in closed_article_snippets):
        print("Article is not publicly available!")
    # specific for PNAS
    

    elif "suppl_file" in str(soup).lower():
        # find the link to the supplementary file
        links = soup.find_all('a')
        for link in links:
            try:
                if "suppl_file" in link.get('href'):
                    # print("There is a supplementary file")
                    supplementary_files.append(link.get('href'))
                    # print(link.get('href'))
            except TypeError:
                pass
        
    for i in filetypes:
        filename = ""
        links = soup.find_all('a')
        for link in links:
            try:
                if "."+i in link.get('href') or "."+i in link.get('title'):
                    # print("There is a file")
                    l = link.get('href')
                    if "http" not in l:
                        l = "https://"+domain + l
                    supplementary_files.append(l)
                    # print(link.get('href'))
            except TypeError:
                pass
    
    return supplementary_files


def download_file(url, output_dir):
    response = requests.get(url)
    filename = url.split('/')[-1]
    if len(filename) > 100:
        filename = filename.split("?")[0]
    with open(os.path.join(output_dir, filename), 'wb') as f:
        f.write(response.content)


def download_osf_file(url, output_dir, file_name):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }
    
    # print(url)
    # print(output_dir)
    # print(file_name)
    try:
        # Send a GET request to the URL
        response = requests.get(url,  headers=headers, stream=True)
        
        # Check if the request was successful
        if response.status_code == 200:
            # Create the output directory if it doesn't exist
            os.makedirs(output_dir, exist_ok=True)
            
            # Define the file path
            file_path = os.path.join(output_dir, file_name)
            
            # Save the content of the response as a file
            with open(file_path, 'wb') as file:
                for chunk in response.iter_content(chunk_size=8192):
                    file.write(chunk)
            
            # print(f"File downloaded successfully and saved as {file_path}")
            return file_path
        else:
            # print(f"Failed to download file. Status code: {response.status_code}")
            return None
    except requests.exceptions.RequestException as e:
        # print(f"An error occurred: {e}")
        return None


def get_data_from_link(link, output_dir="examples/data"):
    print(link, "!!")
    if "osf.io" in link:
        print(link.split("/")[-2])
        zip_ = "https://files.osf.io/v1/resources/"+ link.split("/")[-2] +"/providers/osfstorage/?zip="
        fp = download_osf_file(zip_, output_dir, f'{link.split("/")[-2]}.zip')
        return [fp]
    else:
        output_dir = Path(os.path.expanduser(output_dir))
        output_dir = output_dir.absolute()
        driver = get_webdriver()
        driver.maximize_window()
        print(link)
        
        wait = WebDriverWait(driver, 5)
        driver.get(link)

        real_url = driver.current_url
        # print(real_url)
        domain = real_url.split('/')[2]
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        driver.quit()
        files = []
        for i in filetypes:
            filename = ""
            links = soup.find_all('a')
            for link in links:
                try:
                    if "."+i in link.get('href') or "."+i in link.get('title'):
                        # print("There is a file")
                        l = link.get('href')
                        if "http" not in l:
                            l = "https://"+domain + l
                        files.append(l)
                        # print(link.get('href'))
                except TypeError:
                    pass
        # close the browser
        driver.close()
        # download the files
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)
        # for f in files:
        #     print(f)
        downloaded_files = []
        
        for i in files:
            print("downloading", i)
            
            # example https://www.cell.com/cms/attachment/45d6fbc5-6dd3-47d1-9f2a-26c35d10c983/gr3_lrg.jpg
            filename = i.split("/")[-1]
            
            if len(filename) > 100:
                filename = filename.split("?")[0]
            if filename != "":
                if filename not in downloaded_files:
                    fp = download_osf_file(i, output_dir, filename)
                    downloaded_files.append(fp)
            else:
                print("Invalid filename", i)

            # download_file(i, output_dir)
            # try:
            #     response = requests.get(i, timeout=2)
            # except requests.exceptions.InvalidSchema:
            #     l = i.split("http")[-1]
            #     response = requests.get("http"+l, timeout=2)
                
            # except requests.exceptions.ConnectionError:
            #     continue

            # try:
            #     response.raise_for_status()
            #     filename = i.split('/')[-1]
            #     if "?" in filename:
            #         filename = filename.split("?")[0]
            #     if filename != "":
            #         with open(os.path.join(output_dir, filename), 'wb') as f:
            #             f.write(response.content)
                
            #     if filename != "":
            #         if filename not in downloaded_files:
            #             downloaded_files.append(filename)
            # except requests.exceptions.HTTPError:
            #     pass
            # except requests.exceptions.InvalidSchema:
            #     pass

        return downloaded_files

def DownloadData(data_report, output_dir):
    
    print("")
    print("Attempting to download open data from the web.")
    print("Be advised that due to the nature of different websites, automatic download may not be possible.")
    print("Manual download may be required.")
    print("Please check the output directory for downloaded files.")
    print("")
    
    data_dirs = []
    number_of_files = []
    
    for idx, row in data_report.iterrows():
        data_amount = 0
        data_dir = ""
        # print(row)
        links = []
        for i in ["data_links_keywords", "data_links_web"]:
            try:
                links = links+row[i]
            except KeyError:
                pass
            
        print(links)
        
        title = row["title"]
        # replace all special characters with underscores
        title = "".join([i if i.isalnum() else "_" for i in title])
        data_dir = Path(os.path.expanduser(str(output_dir)+"/"+title))
        
        data_dirs.append(data_dir)
        os.makedirs(data_dir, exist_ok=True)
        
        if len(links) > 0:
            for i in links:
                get_data_from_link(i, data_dir)
        
        # count downloaded files
        for root, dirs, files in os.walk(data_dir):
            for file in files:
                data_amount += 1
        
        number_of_files.append(data_amount)
        
         
    data_report["data_dir"] = data_dirs
    data_report["number_of_files"] = number_of_files
    
    print("Download attempt complete.")
    
    return data_report


if __name__ == '__main__':
    doi = "10.1016/j.tpb.2012.08.002"
    doi = "10.3390/microorganisms10091765"

