from bs4 import BeautifulSoup
import requests 
import pandas as pd
import json
import datetime
from urllib.parse import quote
import urllib3
from .notation import *

# Proxy makers used as fallbacks when direct requests fail
PROXY_MAKERS = [
    lambda u: u,
    lambda u: "https://api.allorigins.win/raw?url=" + quote(u, safe=''),
    lambda u: "https://api.codetabs.com/v1/proxy?quest=" + quote(u, safe=''),
    lambda u: "https://r.jina.ai/http://" + u.replace("https://", "").replace("http://", ""),
]


def _fetch_with_browser(url, timeout: int = 30):
    """Fetch URL using browser automation to bypass Cloudflare protection.
    
    Tries nodriver first (lightweight), then falls back to seleniumbase with UC mode.
    Returns a requests.Response-like object with .text and .json() method.
    """
    # Try nodriver first (modern, lightweight, designed for Cloudflare bypass)
    try:
        import nodriver as uc
        import asyncio
        
        async def fetch():
            browser = await uc.start(headless=True)
            try:
                page = await browser.get(url)
                await asyncio.sleep(10)  # Wait for JS to execute and Cloudflare to solve
                content = await page.get_content()
                await browser.stop()
                return content
            except Exception:
                await browser.stop()
                raise
        
        content = asyncio.run(fetch())
        
        # Extract JSON from HTML if wrapped in <pre> tags
        import re
        json_match = re.search(r'<pre[^>]*>(.+?)</pre>', content, re.DOTALL)
        if json_match:
            content = json_match.group(1).strip()
        
        # Create a response-like object
        class BrowserResponse:
            def __init__(self, text):
                self.text = text
                self.status_code = 200
            def json(self):
                return json.loads(self.text)
            def raise_for_status(self):
                pass
        
        return BrowserResponse(content)
    
    except ImportError:
        pass  # nodriver not installed, try next option
    except Exception:
        pass  # nodriver failed, try next option
    
    # Try seleniumbase with UC mode
    try:
        from seleniumbase import Driver
        import re
        
        driver = Driver(uc=True, headless=True)
        try:
            driver.get(url)
            driver.sleep(10)  # Wait for JS
            content = driver.page_source
            driver.quit()
            
            # Extract JSON from HTML if wrapped in <pre> tags
            json_match = re.search(r'<pre[^>]*>(.+?)</pre>', content, re.DOTALL)
            if json_match:
                content = json_match.group(1).strip()
            
            class BrowserResponse:
                def __init__(self, text):
                    self.text = text
                    self.status_code = 200
                def json(self):
                    return json.loads(self.text)
                def raise_for_status(self):
                    pass
            
            return BrowserResponse(content)
        except Exception:
            driver.quit()
            raise
    
    except ImportError:
        pass  # seleniumbase not installed
    except Exception:
        pass  # seleniumbase failed
    
    raise RuntimeError("Browser automation failed: install 'nodriver' or 'seleniumbase' to bypass Cloudflare protection")


def fetch_url(url, method: str = 'get', data=None, headers=None, timeout: int = 10, verify: bool = True, allow_proxies: bool = True, use_browser: bool = True):
    """Fetch a URL with optional POST data and automatic fallbacks.

    Tries direct request first, then proxies, then browser automation if available.
    Returns a requests.Response on success or raises the last exception on failure.
    
    Args:
        use_browser: If True, will attempt browser automation as final fallback (default: True)
    """
    headers = headers or {"User-Agent": "Mozilla/5.0"}

    last_exc = None
    
    def is_cloudflare_blocked(response):
        """Check if response is a Cloudflare challenge page."""
        if response.status_code != 200:
            return False
        text = response.text[:1000].lower()
        return 'cloudflare' in text and ('challenge' in text or 'just a moment' in text)

    # Try direct request (GET or POST)
    try:
        if method.lower() == 'post':
            r = requests.post(url, data=data, headers=headers, timeout=timeout, verify=verify)
        else:
            r = requests.get(url, headers=headers, timeout=timeout, verify=verify)
        r.raise_for_status()
        if not is_cloudflare_blocked(r):
            return r
        else:
            last_exc = Exception("403 Cloudflare challenge detected")
    except Exception as e:
        last_exc = e

    # If SSL verification failed and we attempted with verify=True, retry without verification
    if verify and any(k in str(last_exc).lower() for k in ['ssl', 'certificate', 'verify']):
        try:
            # Suppress the InsecureRequestWarning when we intentionally disable verification
            import warnings
            from urllib3.exceptions import InsecureRequestWarning
            with warnings.catch_warnings():
                warnings.simplefilter('ignore', InsecureRequestWarning)
                if method.lower() == 'post':
                    r = requests.post(url, data=data, headers=headers, timeout=timeout, verify=False)
                else:
                    r = requests.get(url, headers=headers, timeout=timeout, verify=False)
            r.raise_for_status()
            if not is_cloudflare_blocked(r):
                return r
            else:
                last_exc = Exception("403 Cloudflare challenge detected")
        except Exception as e:
            last_exc = e

    # Try proxies (use GET via proxies since many public proxies only support GET)
    if allow_proxies:
        for maker in PROXY_MAKERS[1:]:
            proxy_url = maker(url)
            try:
                r = requests.get(proxy_url, headers=headers, timeout=timeout)
                r.raise_for_status()
                if not is_cloudflare_blocked(r):
                    return r
            except Exception as e:
                last_exc = e

    # Final fallback: browser automation (only for GET requests to avoid Cloudflare)
    if use_browser and method.lower() == 'get':
        try:
            return _fetch_with_browser(url, timeout=30)
        except Exception as e:
            last_exc = e

    # All attempts failed
    raise last_exc


def get_code(name):
    nottation_code=notation_code()
    for action in nottation_code :
        if action["name"]==name :
            code=action['ISIN']
            return code
def get_valeur(name):
    if name=="MASI":
        print("MASI can't be an argument for this function")
    name_value=notation_value()
    return name_value[name]

def get_data(soup,decode):
    table= json.loads(soup.text.encode().decode(decode))
    row_data=pd.DataFrame(table["result"])
    row_data.columns=["Date","Value","Min","Max","Variation","Volume"]
    date=row_data['Date']
    row_data.drop(['Date'],axis=1,inplace=True)
    row_data.index=date
    return row_data

def intradata(soup,decode):
    table= json.loads(soup.text.encode().decode(decode))
    row_data=pd.DataFrame(table["result"][0])
    index=row_data['labels'].values
    row_data.drop(['labels'],axis=1,inplace=True)
    row_data.index=index
    row_data.columns=["Value"]
    return row_data


def get_index(soup,decode):
    table= json.loads(soup.text.encode().decode(decode))
    row_data=pd.DataFrame(table["result"])
    date=row_data['labels']
    row_data.drop(['labels'],axis=1,inplace=True)
    row_data.index=date.apply(lambda x :datetime.datetime.fromtimestamp(x).date())
    row_data.columns=["Value"]
    return row_data


def produce_data(data,start,end):
    start=pd.to_datetime(start).date()
    end=pd.to_datetime(end).date()
    return data.loc[start:end]

def getTables(soup):
    tabs=['table1',"table6","table7","table4"]
    result=dict()
    for tab in tabs :
        t=soup.find(id=tab).find_all("span")
        t=[ x.get_text() for x in t]
        if tab=='table1':
            a=getTable1(t)
            result["Données_Seance"]=a
        elif tab=="table6":
            a=getTable6(t)
            result["Meilleur_limit"]=a
        elif tab=='table7':
            a=getTable7(t)
            result['Dernieres_Tansaction']=a
        else:
            a=getTable4(t)
            result["Seance_prec"]=a
    return result
def cleanText(text):
    return text.replace(",",".").replace("%","").replace("\xa0","").replace("Â","")
def getTable1(t):
    cols=['Cours','Variation','Ouverture','Plus haut','Plus bas','Cours de cloture veille',
          'Volume','Volume en titres','Capitalisation','Nombre de titres','Devise de cotation']
    t=list(map(cleanText,t))
    a=dict(zip(cols,t))
    return a
def getTable6(t):
    if len(t)>4:
        t=t[1:]
    cols=['Prix achat', 'Quantite_achat', 'Prix de vente', 'Quantite_vente']
    t=list(map(cleanText,t))
    a=dict(zip(cols,t))
    return a
def getTable7(t):
    a=dict()
    i=0
    heure=[]
    prix=[]
    qte=[]
    while i<len(t):
        heure.append(t[i])
        prix.append(t[i+1])
        qte.append(t[i+2].replace("\xa0","").replace("Â",""))
        i+=3
    a["Heure"]=heure
    a["Prix"]=prix
    a["Quantite"]=qte
    return a
def getTable4(t):
    a=dict()
    i=0
    Date = []
    Variation = []
    Cloture = []
    Volume = []
    Ouverture =[]
    Plus_haut = []
    Plus_bas=[]
    while i<len(t):
        Date.append(t[i])
        Variation.append(t[i+1])
        Cloture.append(t[i+2])
        Volume.append(t[i+3])
        Ouverture.append(t[i+4])
        Plus_haut.append(t[i+5])
        Plus_bas.append(t[i+6])
        i+=7    
    a["Date"]=list(map(cleanText,Date))
    a["Variation"]=list(map(cleanText,Variation)) 
    a["Cloture"]=list(map(cleanText,Cloture))
    a["Volume"]=list(map(cleanText,Volume))
    a["Ouverture"]=list(map(cleanText,Ouverture))
    a["Plus_haut"]=list(map(cleanText,Plus_haut))
    a["Plus_bas"]=list(map(cleanText,Plus_bas))
    return a

def getTablesFich(soup):
    tabs=['table4',"table3","table6"]
    result=dict()
    for tab in tabs :
        if tab=='table4':
            t=soup.find(id=tab).find_all("span")
            t=[ x.get_text() for x in t]
            a=getTable4Fich(t)
            result["Info_Societe"]=a
        elif tab=="table3":
            t=soup.find(id="table3").find_all('span')
            t=[x.get_text() for x in t]
            a=getTable3Fich(t)
            result["Actionnaires"]=a
        elif tab=='table6':
            t=soup.find(id="table6").find_all("span")
            t1=soup.find(id="table6").find_all(class_="desc")
            t=[x.get_text().replace("\xa0","").replace("-","") for x in t if x not in t1]
            a=getTable6Fich(t)
            result['Chiffres_cles']=a[0]
            result["Ratio"]=a[1]
    return result

def getTable4Fich(t):
    cols=["Raison_sociale" ,"ISIN","Ticker","Siege_social" ,"Secteur_activité" ,"Commissaire_aux_comptes",
          "Date_de_constitution","Date_introduction","Durée_Exercice_Social","Objet_social"]
    t=t[:4]+t[5:11]
    a=dict(zip(cols,t))
    return a
def getTable3Fich(t):
    a=dict()
    i=0
    while i<len(t):
        a[t[i]]=t[i+1].replace(",",".")
        i+=2
    return a
def getTable6Fich(t):
    a=dict()
    cols_chifr=["Annee","Comptes_consolide","Capital_social","Capitaux_propres","Nombre_titres",
         "Chiffre_Affaires","Resultat_exploitation","Resultat_net"]
    cols_ratio=["Annee","BPA","ROE","Payout","Dividend_yield","PER","PBR"]
    if "Chiffre d'Affaires" in t:
        t.remove("Chiffre d'Affaires")
    if "Résultat d'exploitation" in t:
        t.remove("Résultat d'exploitation")
    annee=[]
    anne=[]
    Comptes_consolide=[]
    Capital_social=[]
    Capitaux_propres=[]
    Nombre_titres=[]
    Chiffre_Affaires=[]
    Resultat_exploitation=[]
    Resultat_net=[]
    BPA=[]
    ROE=[]
    Payout=[]
    Dividend_yield=[]
    PER=[]
    PBR=[]
    chifr=[annee,Comptes_consolide,Capital_social,Capitaux_propres,Nombre_titres,Chiffre_Affaires,
          Resultat_exploitation,Resultat_net]
    ratio=[anne,BPA,ROE,Payout,Dividend_yield,PER,PBR]
    i,j,u=0,0,0
    while i <len(t) and j<len(chifr):
        if i>5 and t[i].replace(",","").isdigit()==False:
            cols_chifr[j]=t[i]
            i+=1
        chifr[j].append(t[i])
        chifr[j].append(t[i+1])
        chifr[j].append(t[i+2])
        i+=3
        j+=1
    while i<len(t) and u<len(ratio):
        ratio[u].append(t[i].replace("\xa0",""))
        ratio[u].append(t[i+1].replace("\xa0",""))
        ratio[u].append(t[i+2].replace("\xa0",""))
        i+=3
        u+=1
    a=dict(zip(cols_chifr,chifr))
    b=dict(zip(cols_ratio,ratio))
    return [a,b]

def getDivi(soup):
    t=soup.find_all(class_="txt_table")[1]
    t=t.find_all("span")
    t=[x.get_text() for x in t ]
    cols=["Annee","Montant_Dividende","Type_Dividende","Date_detachement","Date_paiement"]
    anne=[]
    md=[]
    td=[]
    dd=[]
    dp=[]
    i=0
    while i<len(t) :
        anne.append(t[i])
        md.append(t[i+1])
        td.append(t[i+2])
        dd.append(t[i+3])
        dp.append(t[i+4])
        i+=5
    vals=[anne,md,td,dd,dp]
    return dict(zip(cols,vals))


def getAllIndex(soup):
    indexSumry=getIndexSumry(soup)
    indiceRent=getIndiceRentab(soup)
    indiceDevis=getIndiceDevise(soup)
    indiceDevisF=getIndiceDeviseF(soup)
    indiceSect=getIndiceSect(soup)
    return {"Resume indice":indexSumry,"Indice rentabilite":indiceRent,"Indices en devises":indiceDevis,
           "Indice FTSE":indiceDevisF,"Indices sectoriels":indiceSect}

def getIndexSumry(soup):
    title=soup.find_all(class_='arial11turquoibold')
    values=soup.find_all(class_='arial11noir')
    tabs=["Morocco Stock Index 20","MASI","Casablanca ESG 10","FTSE CSE Morocco 15 Index","FTSE CSE Morocco All-Liquid"]
    result=dict()
    for i in range(len(tabs)):
        cols=[x.get_text().replace("\r","").replace("\n","").replace("  ","") for x in title[i].find_all("td")]
        val=[x.get_text().replace("\xa0","").replace("\n","").replace("Â","") for x in values[i].find_all('td')]
        result[tabs[i]]=dict(zip(cols,val))
    return result

def getIndiceRentab(soup):
    t=soup.find_all(id="Table3")[-1].find_all("span")
    t=[x.get_text().replace("Â","").replace("\xa0","") for x in t]
    cols=["Valeur","Var %","Var % 31/12"]
    return {"MASI-Rentabilite Brut":dict(zip(cols,t[:3])),"MASI-Rentabilite Net":dict(zip(cols,t[3:]))}

def getIndiceDevise(soup):
    t=soup.find(id="Table4").find_all("span")
    t=[x.get_text().replace("Â","").replace("\xa0","") for x in t]
    cols=["Valeur","Var %","Var % 31/12"]
    return {"MASI Dollar":dict(zip(cols,t[:3])),"MASI Euro":dict(zip(cols,t[3:]))}

def getIndiceDeviseF(soup):
    result=dict()
    t=soup.find_all(id="Table2")[-1].find_all("span")
    t=[x.get_text().replace("Â","").replace("\xa0","") for x in t]
    cols=["Valeur","Var %","Var % 31/12"]
    cl=["FTSE Dollar","FTSE Euro","FTSE TR","FTSE All-liquid Dollar","FTSE All-liquid Euro","FTSE All-liquid TR"]
    j=0
    for i in range(len(cl)):
        result[cl[i]]=dict(zip(cols,t[i+j:i+j+3]))
        j+=2
    return result

def getIndiceSect(soup):
    t=soup.find_all(id='arial11bleu')[-1].find_all('span')
    t=[x.get_text().replace("\xa0","").replace("Â","") for x in t]
    i=0
    secteur=[]
    valeur=[]
    var=[]
    var1=[]
    i=0
    while i<len(t):
        secteur.append(t[i])
        valeur.append(t[i+1])
        var.append(t[i+2])
        var1.append(t[i+3])
        i+=4
    vals=[{"Valeur":valeur[i],"Var%":var[i],"Var% 31/12":var1[i]} for i in range(len(secteur))]
    return dict(zip(secteur,vals))

def getPondval(soup):
    t=soup.find_all(class_="arial11gris")
    isin=[]
    inst=[]
    nbrtr=[]
    cours=[]
    flot=[]
    plaf=[]
    capflo=[]
    poid=[]
    vl=[isin,inst,nbrtr,cours,flot,plaf,capflo,poid]
    cl=["Code Isin","Instrument","Nombre de titres","Cours","Facteur flottant","Facteur plafonnement","Capitalisation flottante","Poids"]
    for i in range(1,len(t)):
        vals=[x.get_text().replace("\xa0","").replace("Â","") for x in t[i].find_all('span')]
        for j in range(len(vl)):
            vl[j].append(vals[j])
    return dict(zip(cl,vl))

def getIndiceRecapScrap(soup):
    t=soup.find_all(class_="arial11bleu")
    t=[x.get_text().replace("\n","").replace("\xa0","").replace("\r","").replace("  ","") for x in t][1:]
    b=t[1:6]+t[7:15]+t[16:17]+t[18:21]+[t[22]]+t[24:]
    cl=["Indice","Volume Global","Plus forte hausse","Plus forte baisse"]
    indc=dict()
    hausse=[]
    baisse=[]
    i=0
    while i<len(b) and i<12 :
        indc[b[i]]={"Cours":b[i+1],"Var%":b[i+2],"Var31/12":b[i+3]}
        i+=4
    cols=["Instrument","Cours","Difference","Difference en %"]
    hausse=dict(zip(cols,b[i:i+4]))
    baisse=dict(zip(cols,b[i+4:]))
    volum=soup.find(class_='arial18vertfluo').get_text().replace('\n','').replace("\xa0","")
    result=dict(zip(cl,[indc,volum,hausse,baisse]))
    return result