# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/02-parser.ipynb (unless otherwise specified).

__all__ = ['Driver', 'CategoryScrapper', 'get_category_df_from_page', 'update_page_num', 'run', 'SingleItemScrapper',
           'get_params_df', 'get_sold_qty', 'get_seller_params', 'get_description', 'get_rates_vs_percentage',
           'get_timestamp_from_row', 'get_first_feedback_timestamp', 'run', 'MultipleItemScrapper', 'run',
           'ShopstatScrapper', 'authorize', 'get_single_item_df', 'run']

# Cell

from selenium import webdriver
from selenium.webdriver.common.by import By

from bs4 import BeautifulSoup
from time import sleep

import fastcore.basics as fcb
import pandas as pd
from .core import logger
from urllib import parse
import numpy as np

import re

from .postgres import PostgresConnector

from tqdm import tqdm

import psutil

from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

# Cell

class Driver:
    def __init__(self):

        # create options for the driver:
        options = webdriver.ChromeOptions()
        options.add_argument('--headless')
        options.add_argument("--disable-extensions")
        options.add_argument("--disable-notifications")
        options.add_argument("--disable-gpu")
        options.add_argument("--headless")
        options.add_argument("--no-sandbox")
        options.add_argument("--disable-dev-shm-usage")
        options.add_argument("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.82 Safari/537.36")


        # start driver:
        self.driver = webdriver.Chrome(options=options)

        # self.driver = uc.Chrome(options=options)
        logger.info(f"SUCCESS: Chrome driver was created")

    def get_driver(self):
        return self.driver

# Cell

class CategoryScrapper:
    def __init__(self, driver, category_url):

        # create driver attribute:
        self.driver = driver

        # create category_url attribute:
        self.category_url = category_url

# Cell

@fcb.patch_to(CategoryScrapper)
def get_category_df_from_page(self, page_url):

    """
    METHOD FOR SCRAPPING ALL 100 ITEMS FROM page_url

    page_url - some page of category search

    Returns: pd.DataFrame
    """

    # GET DRIVER:
    driver = self.driver
    driver.get(page_url)

    # WAIT THE PAGE TO UPLOAD UP TO 60 SECS:
    WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "product-card-list")))

    # SCROLL:
    scroll_height = 500 # height to scroll down

    # scroll down until can get 100 items:
    item_wrappers = []
    while len(item_wrappers)!=100:
        try:
            scroll_height +=400
            driver.execute_script(f"window.scrollTo(0, {scroll_height});")
            sleep(0.3)
            pageSource = driver.page_source
            bs = BeautifulSoup(pageSource, 'html.parser')
            item_wrappers = bs.findAll('div', {'class':'product-card j-card-item'})
        except:
            pass

    # get items dataframe:
    url_list = []
    name_list = []
    price_list = []
    price_with_discount_list = []
    feedbacks_list = []
    rate_list = []

    # get data for each of 100 items:
    for item_wrapper in item_wrappers:

        # get url:
        url = item_wrapper.find('a', {'class':'product-card__main j-card-link'})['href']

        # get name:
        name = item_wrapper.find('span', {'class':'goods-name'}).get_text().replace(' / ','')

        # get price and price_with_discount:
        try:
            # if item has price (crossed out price):
            price_with_discount = item_wrapper.find('ins', {'class':'price__lower-price'}).get_text().replace(u'\xa0', '').strip()[:-1:]
            price_with_discount = int(price_with_discount)

            price = item_wrapper.find('del').get_text().replace(u'\xa0', '').strip()[:-1:]
            price = int(price)

        except:
            # if item doesn't have price_with_discount:
            price_with_discount = item_wrapper.find('span', {'class':'price__lower-price'}).get_text().replace(u'\xa0', '').strip()[:-1:]
            price_with_discount = int(price_with_discount)

            price = None


        # get feedbacks:
        feedbacks = item_wrapper.find('span', {'class':'product-card__count'}).get_text().replace(u'\xa0', '')
        feedbacks = int(feedbacks)

        # get rate:
        for num in [0, 1, 2, 3, 4, 5]:
            if item_wrappers[0].find('span', {'class':f'product-card__rating stars-line star{num}'}) is not None:
                rate = num
                break

        # append values to lists:
        url_list.append(url)
        name_list.append(name)
        price_list.append(price)
        price_with_discount_list.append(price_with_discount)
        feedbacks_list.append(feedbacks)
        rate_list.append(rate)

    df = pd.DataFrame({'url':url_list,
                       'name':name_list,
                       'price':price_list,
                       'price_with_discount':price_with_discount_list,
                       'feedbacks':feedbacks_list,
                       'rate':rate})

    # extract code of items (артикулы):
    df['code'] = df.url.apply(lambda x: re.search("\/catalog\/(.*)\/detail", x).group(1))

    # remove column 'url':
    df = df.drop('url', axis=1)

    return df

# Cell

@fcb.patch_to(CategoryScrapper)
def update_page_num(self, category_url, page_num):

    """
    METHOD FOR CHANGING PAGE NUMBER IN URL

    category_url - any page of category (search request)

    """

    parsed_url = parse.urlparse(category_url)
    query_params = parse.parse_qs(parsed_url.query)
    query_params['page'] = [page_num]
    parsed_url = parsed_url._replace(query=parse.urlencode(query_params, doseq=True))
    updated_url = parse.urlunparse(parsed_url)

    return updated_url

# Cell

@fcb.patch_to(CategoryScrapper)
def run(self,
        n,
        postgres_creds,
        save_to_postgres=False):

    """
    METHOD FOR GETTING ITEMS_DF FROM FIRST n (INCLUDING) PAGES FROM category_url

    category_url - any page of category (search request)

    """

    # get category_url:
    category_url = self.category_url


    # for first n pages of category_url - scrap all items:
    dfs = []
    for page_num in tqdm(range(1, n+1)):
        logger.info(f"Started scrapping page {page_num}")
        page_url = self.update_page_num(category_url, page_num)
        df = self.get_category_df_from_page(page_url)
        dfs.append(df)
        logger.info(f"SUCCESS: finished scrapping page {page_num}")
        logger.info(f"CPU: {psutil.cpu_percent():.0f}%, RAM: {psutil.virtual_memory().percent:.0f}%")

    # create a dataframe:
    category_df = pd.concat(dfs)
    category_df['category_url'] = self.category_url
    category_df = category_df[['category_url', 'code', 'name', 'price', 'price_with_discount', 'feedbacks', 'rate']]

    if save_to_postgres:
        pc = PostgresConnector(**postgres_creds)
        pc.check_connection()
        pc.save(df=category_df, table_name='category_table', schema='main_schema', if_table_exists='append')
        logger.info(f"SUCCESS: saved category_df to Postgres")

    return category_df

# Cell

class SingleItemScrapper:
    def __init__(self, driver, code):

        # create code (артикул) attribute:
        self.code = code


        ## GET ITEM'S PAGE BEAUTIFULSOUP OBJECT:

        # create item_page_url from code (артикул):
        item_page_url = f'https://www.wildberries.ru/catalog/{code}/detail.aspx'

        # wait page up to 60 secs to upload:
        driver.get(item_page_url)
        WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "product-params__caption"))) # changed here

        # get item's page BeautifulSoup object:
        pageSource = driver.page_source
        self.item_page_bs = BeautifulSoup(pageSource, 'html.parser')


        ## GET FEEDBACKS PAGE BEAUTIFULSOUP OBJECT:

        # check if the qty of feedbacks is 0:
        feedbacks_qty = self.item_page_bs.find('span', {'class':'product-review__count-review'}).get_text()
        feedbacks_qty = re.sub(r'[^\d]', '', feedbacks_qty) # removing these patterns
        self.feedbacks_qty = int(feedbacks_qty)

        # get feedbacks page url:
        item_page_url_parts = item_page_url.split('/')
        item_page_url_parts[-1] = 'feedbacks'
        feedbacks_url = '/'.join(item_page_url_parts)


        # get feedbacks page BeautifulSoup object if feedbacks qty != 0:

        if self.feedbacks_qty != 0:

            driver.get(feedbacks_url)

            # wait page up to 60 secs to upload:
            WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "sorting__selected")))

            # click button to sort feedbacks by date:
            button = driver.find_element(By.CLASS_NAME, value='sorting__selected')
            driver.execute_script("arguments[0].click();", button)

            # wait page up to 60 secs to upload:
            WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CLASS_NAME, "feedback__info")))

            pageSource = driver.page_source
            self.feedbacks_page_bs = BeautifulSoup(pageSource, 'html.parser')
        else:
            pass

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_params_df(self):

    """
    METHOD FOR EXTRACTING PARAMETERS FROM TABLES ON PAGE

    Returns: pd.DataFrame
    """

    # get BeautifulSoup object:
    bs = self.item_page_bs


    param_name_list  = []
    param_value_list = []

    # get list of tables' HTMLs:
    param_tables = bs.findAll('table', {'class':"product-params__table"})

    # for each table extract params' names and append to param_name_list and params' values and append to param_value_list
    # (thus we unite all param tables on page - into one):
    for param_table in param_tables:

        # get param names:
        param_names = param_table.findAll('th', {'class':'product-params__cell'})
        for param_name in param_names:
            param_name_list.append(param_name.get_text())

        # get param values:
        param_values = param_table.findAll('td', {'class':'product-params__cell'})
        for param_value in param_values:
            param_value_list.append(param_value.get_text())


    # try to extract param 'color' and append it to dataframe:
    try:
        color = bs.find('span', {'class':"color"}).get_text()
        param_name_list.append('Цвет')
        param_value_list.append(color)
    except:
        param_name_list.append('Цвет')
        param_value_list.append(np.nan)

    # get dataframe of params and values and drop duplicates (same params from different tables):
    params_df = pd.DataFrame({'param':param_name_list, 'value':param_value_list}).drop_duplicates()

    return params_df

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_sold_qty(self):

    """
    METHOD FOR EXTRACTING SOLD_QTY ON PAGE

    Returns: pd.DataFrame
    """

    # get BeautifulSoup object:
    bs = self.item_page_bs

    # try to extract sold_qty (not any item must have a sold_qty - for example new items do not)
    try:
        sold_qty = bs.find('span', {'data-link': "{include tmpl='productCardOrderCount' ^~ordersCount=selectedNomenclature^ordersCount}"}).get_text()\
                     .replace('\xa0', '').split(' ')[2]
        sold_qty = int(sold_qty)

        return pd.DataFrame({'param':['sold_qty'], 'value':[sold_qty]})


    except:
        return pd.DataFrame({'param':['sold_qty'], 'value':[0]}) # no items sold yet

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_seller_params(self):

    """
    METHOD FOR EXTRACTING SELLER'S FEEDBACK AND SELLER'S RATE ON PAGE

    Returns: pd.DataFrame
    """

    # get BeautifulSoup object:
    bs = self.item_page_bs

    # get seller's feedbacks qty:
    seller_feedbacks = bs.find('span', {'class':'seller-info__review'}).get_text()
    seller_feedbacks = seller_feedbacks.replace(' отзывов на товары','').replace(' ','')
    seller_feedbacks = int(seller_feedbacks)

    # get seller's rate:
    seller_rate = bs.find('span', {'class':'address-rate-mini'}).get_text()
    seller_rate = float(seller_rate)

    return pd.DataFrame({'param':['seller_rate', 'seller_feedbacks'], 'value':[seller_rate, seller_feedbacks]})

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_description(self):

    """
    METHOD FOR EXTRACTING DESCRIPTION ON PAGE

    Returns: pd.DataFrame
    """

    # get BeautifulSoup object:
    bs = self.item_page_bs

    # get description:
    description = bs.find('p', {'class':'collapsable__text'}).get_text()

    return pd.DataFrame({'param':['description'], 'value':[description]})

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_rates_vs_percentage(self):

    """
    METHOD FOR EXTRACTING TABLE OF FEEDBACKS'S RATES VS. % OF FEEDBACKS WITH THIS RATE
    (SHOWN ON FEEDBACKS PAGE IN THE TOP)

    Returns: pd.DataFrame
    """
    # check if feedbacks qty != 0:
    if self.feedbacks_qty==0:
        return pd.DataFrame({'param':[f'percentage_of_feedbacks_with_rate_{i}' for i in range(1, 6)], 'value':[np.nan]*5}) # nan values


    # get BeautifulSoup object:
    bs = self.feedbacks_page_bs

    # get list of % of feedbacks with rates 1, 2, 3, 4, 5:
    rates = []
    for rate in [5, 4, 3, 2, 1]:
        if bs.find('span', {'class':f'rating-product__sorting-stars stars-line star{rate}'}) is not None:
            rates.append(rate)

    percentages = []
    for percentage_html in bs.findAll('p', {'class':'rating-product__review'})[1::]:
        percentage = percentage_html.get_text().replace('%','')
        percentage = int(percentage)
        percentages.append(percentage)

    # create dataframe:
    rates_vs_percentage = pd.DataFrame({'param':rates, 'value':percentages})
    rates_vs_percentage['param'] = rates_vs_percentage['param'].apply(lambda x: f'percentage_of_feedbacks_with_rate_{x}')

    return rates_vs_percentage

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_timestamp_from_row(self, first_feedback_date_string):

    """
    METHOD FOR CONVERTING STRING OF FIRST FEEDBACK'S DATE INTO pd.Timestamp

    Returns: pd.Timestamp
    """

    # check if date is today or yesterday:
    if 'вчера' in first_feedback_date_string.lower():
        return pd.Timestamp.now().floor('D') - pd.Timedelta(days=1)
    elif 'сегодня' in first_feedback_date_string.lower():
        return pd.Timestamp.now().floor('D')

    # remove hour and minutes from string:
    first_feedback_date_string = first_feedback_date_string.split(',')[0].replace(' ','-')

    # if there's no year in string - add current year:
    if first_feedback_date_string.count('-')==1:
        current_year = pd.Timestamp.now().year
        first_feedback_date_string = f'{first_feedback_date_string}-{current_year}'

    months = {'января': 'Jan', 'февраля': 'Feb', 'марта': 'Mar', 'апреля': 'Apr',
              'мая': 'May', 'июня': 'Jun', 'июля': 'Jul', 'августа': 'Aug',
              'сентября': 'Sep', 'октября': 'Oct', 'ноября': 'Nov', 'декабря': 'Dec'}

    for month, short_form in months.items():
        first_feedback_date_string = re.sub(month, short_form, first_feedback_date_string)

    return pd.Timestamp(first_feedback_date_string)

# Cell

@fcb.patch_to(SingleItemScrapper)
def get_first_feedback_timestamp(self):

    """
    METHOD FOR EXTRACTING FIRST FEEDBACK'S pd.Timestamp FROM PAGE

    Returns: pd.DataFrame
    """
    # check if feedbacks qty != 0:
    if self.feedbacks_qty==0:
        return pd.DataFrame({'param':['first_feedback_timestamp'], 'value':[np.nan]}) # nan value

    # get BeautifulSoup object:
    bs = self.feedbacks_page_bs

    # get first feedbacks's date:
    first_feedback_date_string = bs.find('span', {'class':'feedback__date hide-mobile'}).get_text()

    first_feedback_timestamp = self.get_timestamp_from_row(first_feedback_date_string)

    return pd.DataFrame({'param':['first_feedback_timestamp'], 'value':[first_feedback_timestamp]})

# Cell

@fcb.patch_to(SingleItemScrapper)
def run(self):

    """
    METHOD FOR EXTRACTING FULL DATA FOR ITEM

    Returns: pd.DataFrame
    """

    logger.info(f"Started scrapping code {self.code}")

    params_df = self.get_params_df()
    sold_qty = self.get_sold_qty()
    first_feedback_timestamp = self.get_first_feedback_timestamp()
    rates_vs_percentage = self.get_rates_vs_percentage()
    description = self.get_description()

    full_param_df = pd.concat([params_df, sold_qty, first_feedback_timestamp, rates_vs_percentage, description])
    full_param_df['code'] = self.code
    full_param_df = full_param_df[['code', 'param', 'value']]

    logger.info(f"SUCCESS: finished scrapping code {self.code}")

    return full_param_df

# Cell

class MultipleItemScrapper:
    def __init__(self, driver, category_url):

        # create driver attribute:
        self.driver = driver

        # create category_url attribute:
        self.category_url = category_url

# Cell

@fcb.patch_to(MultipleItemScrapper)
def run(self, code_list, postgres_creds, save_to_postgres=False):

    """
    METHOD FOR SCRAPPING ALL ITEMS FROM code_list TO GET DATA FROM THEIR PAGES.

    Returns: pd.DataFrame
    """

    logger.info(f"SUCCESS: started scrapping multiple ({len(code_list)}) items")

    dfs = []

    for code in tqdm(code_list):
        sis = SingleItemScrapper(self.driver, code)
        item_param_df = sis.run()
        dfs.append(item_param_df)
        logger.info(f"CPU: {psutil.cpu_percent():.0f}%, RAM: {psutil.virtual_memory().percent:.0f}%")

    items_full_df = pd.concat(dfs)

    # add column category_url:
    items_full_df['category_url'] = self.category_url

    items_full_df = items_full_df[['category_url', 'code', 'param', 'value']]

    logger.info(f"SUCCESS: finished scrapping multiple ({len(code_list)}) items")

    if save_to_postgres:
        pc = PostgresConnector(**postgres_creds)
        pc.check_connection()
        pc.save(df=items_full_df, table_name='items_table', schema='main_schema', if_table_exists='append')
        logger.info(f"SUCCESS: saved items_full_df to Postgres")

    return items_full_df

# Cell

class ShopstatScrapper:
    def __init__(self, driver, email, password):

        # create driver attribute:
        self.driver = driver

        # create creds attributes:
        self.email = email
        self.password = password

# Cell
@fcb.patch_to(ShopstatScrapper)
def authorize(self):

    """
    METHOD FOR AUTHORIZING ON SHOPSTATS.

    Returns: None
    """

    # enter authorization page:
    driver = self.driver
    authorization_url = 'https://app.shopstat.ru/auth/login-by-email'
    driver.get(authorization_url)

    # wait page up to 60 secs to upload authorization page:
    WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CSS_SELECTOR, "input[name='email']")))

    # authorize:
    email_input = driver.find_element(By.CSS_SELECTOR, value="input[name='email']")
    password_input = driver.find_element(By.CSS_SELECTOR, value="input[name='password']")

    email_input.send_keys(self.email)
    password_input.send_keys(self.password)

    submit_button = driver.find_element(By.CSS_SELECTOR, "button[type='submit']")
    submit_button.click()

    logger.info(f"SUCCESS: authorized at shopstat.ru")

    # wait up to 60 secs until redirect the main page:
    WebDriverWait(driver, 60).until(EC.url_to_be("https://app.shopstat.ru/"))

    # wait up to 60 secs until main page to upload:
    WebDriverWait(driver, 60).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".MuiCircularProgress-root"))) # graph on page
    logger.info(f"SUCCESS: entered main page at shopstat.ru")

# Cell
@fcb.patch_to(ShopstatScrapper)
def get_single_item_df(self, code):

    """
    METHOD FOR DOWNLOADING SINGLE ITEM'S SOLD_QTY.

    Returns: pd.DataFrame
    """

    logger.info(f"Started scrapping code {code} at shopstat.ru")

    # enter authorization page:
    driver = self.driver

    # get url of item's page on Shopstat.ru and enter it:
    item_url = f'https://app.shopstat.ru/product/wildberries/{code}'
    driver.get(item_url)

    # wait page up to 60 secs to upload either exit button or item table: (if exit banner appears it would only waste 5 secs once trying to wait for item_table_locator to upload)
    exit_button_locator = (By.CSS_SELECTOR, "div.------src-components-onboarding-OnboardingContent__text--VMAOC")
    item_table_locator = (By.XPATH, '//*[@id="root"]/div[2]/div/div[1]/div/section/div[3]/div[2]/div/div/div[5]/div/div/div/div[2]/div[1]/div[2]/div/div/div/div/div/div[3]/div[1]/div/table')

    try:
        WebDriverWait(driver, 60).until(EC.presence_of_element_located(item_table_locator))
    except:
        WebDriverWait(driver, 60).until(EC.presence_of_element_located(exit_button_locator))

    # try to exit banner if exists:
    try:
        exit_button = driver.find_element(By.CSS_SELECTOR, value="button.MuiButtonBase-root.MuiIconButton-root.MuiIconButton-sizeMedium.------src-components-onboarding-OnboardingContent__close--cLp_I.css-maoza2[tabindex='0'][type='button']")
        exit_button.click()
        logger.info(f"Clicked on exit button")
        # wait click action to load:
        sleep(1)
        # wait page up to 60 secs to upload item table:
        WebDriverWait(driver, 60).until(EC.presence_of_element_located(item_table_locator))
    except:
        pass

    # get item's page BeautifulSoup object:
    pageSource = driver.page_source
    bs = BeautifulSoup(pageSource, 'html.parser')

    # for each row of data table - get date and sold_qty:
    date_list = []
    sold_qty_list = []

    for row in bs.findAll('tr', {'class':'MuiTableRow-root MuiTableRow-hover ------src-components-common-ui-Table__row--_e3xe css-e42jjo'})[:30:]:

        date = row.findAll('td')[0].get_text()
        date = pd.to_datetime(date, format='%d.%m.%Y')
        date_list.append(date)

        sold_qty = row.findAll('td')[1].get_text().replace('\xa0шт','')
        sold_qty = int(sold_qty)
        sold_qty_list.append(sold_qty)

    # create dataframe:
    date_vs_sold_qty = pd.DataFrame({'code':code, 'date':date_list, 'sold_qty':sold_qty_list})

    logger.info(f"SUCCESS: finished scrapping code {code} at shopstat.ru")

    return date_vs_sold_qty

# Cell
@fcb.patch_to(ShopstatScrapper)
def run(self, code_list, postgres_creds, save_to_postgres=False):

    """
    METHOD FOR DOWNLOADING MULTIPLE ITEMS' SOLD_QTY.

    Returns: pd.DataFrame
    """
    # authorize:
    self.authorize()

    # for code in codes - scrap its sold_qty:
    dfs = []
    for code in tqdm(code_list):
        df = self.get_single_item_df(code)
        dfs.append(df)
        logger.info(f"CPU: {psutil.cpu_percent():.0f}%, RAM: {psutil.virtual_memory().percent:.0f}%")

    multiple_items_df = pd.concat(dfs)

    if save_to_postgres:
        pc = PostgresConnector(**postgres_creds)
        pc.check_connection()
        pc.save(df=multiple_items_df, table_name='shopstat_table', schema='main_schema', if_table_exists='append')
        logger.info(f"SUCCESS: saved to shopstat_table in Postgres")

    return multiple_items_df