import requests
import pandas as pd
import multiprocessing as mp
from concurrent.futures import ThreadPoolExecutor
from json import loads
from bs4 import BeautifulSoup

# importing the sys module
import sys

# the setrecursionlimit function is
# used to modify the default recursion
# limit set by python. Using this,
# we can increase the recursion limit
# to satisfy our needs

sys.setrecursionlimit(10 ** 6)


def get_description_for_ticker(
    ticker: str,
) -> str:
    """
    set of functionality
    """
    # grab ticker return desc
    # do on google colab
    # https://stackoverflow.com/questions/56987872/parallelize-pandas-column-update
    symbol = grab_symbol_for_ticker(ticker)
    if symbol == None:
        return ""
    return company_description_by_ticker(symbol)


def get_mig_report(
    filename: str = "", exchange: str = "TSX", return_df: bool = False
) -> str:
    """
    Description:
      Gets excel spreadsheet from tsx api programatically.
      See for more flexibility :func:`dl_tsx_xlsx <cad_tickers.exchanges.tsx.dl_tsx_xlsx>`

    Parameters:
      filename: Name of the file to be saved
      exchanges: TSX, TSXV
      return_df: Return a pandas dataframe

    Returns:
      filePath: returns path to file or dataframe

    See ://stackoverflow.com/questions/13567507/passing-csrftoken-with-python-requests

    """
    tsx_url = "https://api.tmxmoney.com/en/migreport/search"
    client = requests.session()
    # Retrieve the CSRF token first
    client.get(tsx_url)  # sets cookie
    if "csrftoken" in client.cookies:
        # Django 1.6 and up
        csrftoken = client.cookies["csrftoken"]
    else:
        # older versions
        csrftoken = client.cookies["csrf"]
    login_data = dict(report_type="excel", csrfmiddlewaretoken=csrftoken)
    if exchange in ["TSX", "TSXV"]:
        login_data["exchanges"] = exchange

    r = client.post(
        "https://api.tmxmoney.com/en/migreport/search",
        data=login_data,
        headers=dict(Referer=tsx_url),
    )
    resp_data = r.content
    if return_df == True:
        return pd.read_excel(resp_data)
    else:
        resp_headers = r.headers
        if "text/html" in resp_headers["Content-Type"]:
            return None
        elif resp_headers["Content-Type"] == "application/ms-excel":
            print(f"Downloaded data from {tsx_url} for {exchange} to {filename}")
            with open(filename, "wb") as s:
                s.write(resp_data)
            return filename
        else:
            return None


def grab_symbol_for_ticker(ticker: str) -> str:
    """
    Description:
      Grabs the first symbol from ticker data
      all symbols should lead to valid webpages for data scrapping.

    Parameters:
      ticker: string representing the stock ticker

    Returns:
      symbol: string - searchable string in the quotemedia api or empty string
    """

    if ticker == None or ticker == "":
        return ""

    ticker_data = lookup_symbol_by_ticker(ticker)
    if len(ticker_data) == 0:
        # splice name and try again
        # remove everything past dot
        clean_ticker = ticker.split(".", 1)[0]
        ticker_data = lookup_symbol_by_ticker(clean_ticker)

        # Ticker likely has no data
        # see NTY
        if len(ticker_data) == 0:
            return ""
    return ticker_data[0].get("symbol")


def add_descriptions_to_df_pp(df: pd.DataFrame, max_workers: int = 16) -> pd.DataFrame:
    """
    Description: fetch descriptions for tickers in parallel
    noticable speedup uses thread pool which should be faster

    Input:
      df: dataframe containing tickers
    Returns:
      df: updated dataframe with a descriptions if available
    """
    df["description"] = ""
    tickers = df["Ticker"].tolist()
    with ThreadPoolExecutor(max_workers=max_workers) as tpe:
        iterables = tpe.map(get_description_for_ticker, tickers)
        descriptions = list(iterables)
    df["description"] = descriptions
    return df

# https://stackoverflow.com/questions/56987872/parallelize-pandas-column-update
def add_descriptions_to_df(df) -> pd.DataFrame:
    """
    Description: single process solution to fetching descriptions

    Input:
      df: dataframe containing tickers
    Returns:
      df: updated dataframe with a descriptions if available
    """
    df["description"] = ""
    for index, row in df.iterrows():
        ticker = df.at[index, "Ticker"]
        # grab ticker return desc
        # do on google colab
        # https://stackoverflow.com/questions/56987872/parallelize-pandas-column-update
        symbol = grab_symbol_for_ticker(ticker)
        if symbol == None:
            pass
        description = company_description_by_ticker(symbol)
        df.at[index, "description"] = description
    return df


def company_description_by_ticker(ticker) -> str:
    """

    Description: Grabs searchable ticker from quotemedia using tmx ticker

    Input:
      ticker: string
    Returns:
      df:  updated dataframe with a descriptions if available
    """
    # get lookup symbol
    search_symbol = grab_symbol_for_ticker(ticker)
    if search_symbol == "":
        return ""
    params = {"qm_symbol": search_symbol}
    # // //*[@id="pane-news"]/div/div/div[1]/div/div[1]/div[2]/section[1]/p

    search_url = f"https://web.tmxmoney.com/company.php"
    r = requests.get(search_url, params=params)
    html_content = r.text
    # Parse the html content
    soup = BeautifulSoup(html_content, "lxml")
    # selector copied from chrome https://money.tmx.com/en/quote/ZMD.H
    description_selector = "#pane-news > div > div > div.col-md-9 > div > div:nth-child(1) > div.tmx-panel-body.pt-0 > section:nth-child(1) > p"
    description_tags = soup.select(description_selector)
    if len(description_tags) > 0:
        description_tag = description_tags[0]
        # grab contents from desscription tag
        if len(description_tag.contents) == 0:
            return ""
        else:
            return description_tag.contents[0]
    # company does not have profile
    else:
        return ""


def lookup_symbol_by_ticker(ticker: str) -> list:
    """

    Description: Returns search array dictionary for tickers

    .. note::

      sometimes the name of the ticker in the xlsx
      sheet is off slightly and we need to find the "real ticker".
      Uses standard api (not graphql) to grab tickers

      Example searchpoint is https://app.quotemedia.com/lookup?callback=tmxtickers&q=zmd&limit=5&webmasterId=101020

    See `Tmx Graphql <https://app-money.tmx.com/graphql>`_  and the new `tmx site <https://money.tmx.com/en/>`_

    Input:
      ticker: tmx ticker

    Output:
      quote_data: list of ticker metadata
    """
    if ticker == None:
        return []
    callback = "tmxtickers"
    # Theses values can be hardcoded
    params = {"callback": callback, "limit": 5, "webmasterId": 101020, "q": ticker}
    quote_lookup_url = "https://app.quotemedia.com/lookup"
    r = requests.get(quote_lookup_url, params=params)
    quote_data = r.text

    if quote_data == None:
        raise Exception(
            "No data returned, check if api is depreciated or contact author for help."
        )
    s_idx = quote_data.find("(")
    e_idx = quote_data.rfind(")")

    if quote_data == f"{callback}();":
        return []
    # All symbols returned all valid
    try:
        data = loads(quote_data[s_idx + 1 : e_idx])
        return data
    except Exception as e:
        print(ticker)
        print("fail to run")
        print(e)
        exit(1)


def dl_tsx_xlsx(filename: str = "", **kwargs) -> str:
    """
    Description: Gets excel spreadsheet from the tsx api using programatically

    .. note::

      Replicates api calls in TSX discover tool with all parameters.
      See `migreport search <https://api.tmxmoney.com/en/migreport/search>`_
      Note that not all parameters are documented and/or limited validation

    Parameters:
      filename: Name of the file to be saved

    Kwargs:
      * exchanges (string): TSX, TSXV
      * marketcap (string): values from 0 to specified value
      * sectors (string): cpc, clean-technology, closed-end-funds, technology

    Returns:
      data - returns path to file or pandas dataframe

      pd.DataFrame

      ===========   =====================================================================
      Ex.           Exchange ticker in TSXV, TSX
      Name          Full name of ticker
      Ticker        Symbol usually 4 characters or less
      QMV($)        Quoted Market Value, I assume this is based on the "currency".
      HQ Region     Headquarters region usually a country (need to double check)
      HQ Location   Usually a province or state
      Sector        Main sector, technology
      Sub Sector    Sub Sector
      ===========   =====================================================================

    .. note
       if this package gets popular enough, I will add more documentation

    See `passing csrftoken <https://stackoverflow.com/questions/13567507/passing-csrftoken-with-python-requests>`_
    """

    tsx_url = "https://api.tmxmoney.com/en/migreport/search"
    client = requests.session()
    # Retrieve the CSRF token first
    client.get(tsx_url)  # sets cookie
    if "csrftoken" in client.cookies:
        # Django 1.6 and up
        csrftoken = client.cookies["csrftoken"]
    else:
        # older versions
        csrftoken = client.cookies["csrf"]
    search_data = dict(report_type="excel", csrfmiddlewaretoken=csrftoken)

    for k, v in kwargs.items():
        if k == "exchanges":
            if k in ["TSX", "TSXV"]:
                search_data["exchanges"] = k
        elif k == "marketcap":
            if k in [
                "0-50",
                "50-250",
                "250-500",
                "500-1000",
                "1000-9999999999999999999999",
            ]:
                search_data["marketcap"] = k
        else:
            search_data[k] = v

    r = client.post(tsx_url, data=search_data, headers=dict(Referer=tsx_url))
    resp_headers = r.headers
    if "text/html" in resp_headers["Content-Type"]:
        return None
    elif resp_headers["Content-Type"] == "application/ms-excel":
        resp_data = r.content
        if filename == "":
            return pd.read_excel(resp_data)
        else:
            with open(filename, "wb") as f_:
                f_.write(resp_data)
            return filename
    else:
        return None


# Add endpoint to query search parameters for tsx spreadsheets
if __name__ == "__main__":
    from datetime import datetime

    start_time = datetime.now()
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-f", "--file", help="file name (default: %(default)s)'", default="tsx.xlsx"
    )
    parser.add_argument(
        "-e",
        "--exchange",
        default="TSX",
        const="TSX",
        nargs="?",
        choices=("TSX", "TSXV", "None", "All"),
        help="TSX, TSXV, All (default: %(default)s)",
    )
    args = parser.parse_args()
    df = dl_tsx_xlsx(sectors="technology")
    df = add_descriptions_to_df_pp(df)
    print(df)
    end_time = datetime.now()
    print(end_time - start_time)
