# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/04-hh_parser.ipynb (unless otherwise specified).

__all__ = ['HeadHunterScrapper', 'get_urls_from_professional_role', 'get_url_vs_responses', 'get_vacancies_info']

# Cell

import pandas as pd
from random import choice, uniform
import requests
import numpy as np
import fastcore.basics as fcb
from .core import logger
from .postgres import PostgresConnector
import time
from tqdm import tqdm
import re

# Cell

class HeadHunterScrapper:
    def __init__(self, base_url='https://api.hh.ru/vacancies'):

        # self.proxies_list = proxies_list
        self.base_url = base_url

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_urls_from_professional_role(self, professional_role, per_page=100, pages=20, sleep=(1,3)):

    logger.info(f'SUCCESS: started scrapping vacancies URLs for professional_role = {professional_role}')

    # get API's base url:
    base_url = self.base_url

    # get vacancies' urls of entered professional_role:
    urls = []
    responses = []
    ids = []
    for page in range(pages):

        logger.info(f'SUCCESS: started scrapping page {page}')

        # make request:
        params = {'professional_role':professional_role, 'area':1, 'per_page':per_page, 'page':page, 'responses_count_enabled':True} # Moscow
        # json = requests.get(base_url, params=params, proxies=proxies).json()
        json = requests.get(base_url, params=params).json()

        # break if there're no more vacancies of professional_role:
        if len(json['items'])==0:
            break

        # get list of vacancies' urls and append it:
        vacancy_urls      = pd.Series(json['items']).apply(lambda x: x['url']).to_list()
        vacancy_responses = pd.Series(json['items']).apply(lambda x: x['counters']['responses']).to_list() # кол-во откликов
        vacancy_ids       = pd.Series(json['items']).apply(lambda x: x['id']).to_list()

        urls      += vacancy_urls
        responses += vacancy_responses
        ids       += vacancy_ids

        # sleep:
        min_sleep = sleep[0]
        max_sleep = sleep[1]
        time.sleep(uniform(min_sleep,max_sleep))

        logger.info(f'SUCCESS: finished scrapping page {page}')

    # create dataframe:
    vacancy_url_df = pd.DataFrame({'professional_role':professional_role, 'url':urls, 'responses':responses, 'id':ids})

    logger.info(f'SUCCESS: finished scrapping vacancies URLs for professional_role = {professional_role}')

    return vacancy_url_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_url_vs_responses(self, professional_roles, per_page=100, pages=20, sleep=(1,3), postgres_creds=None, save_to_postgres=False):

    logger.info(f'SUCCESS: started scrapping vacancies URLs for professional_roles = {professional_roles}')

    dfs = []
    for professional_role in tqdm(professional_roles):

        # try up to 5 times:
        for attempt in range(5):
            try:
                df = self.get_urls_from_professional_role(professional_role, per_page=per_page, pages=pages, sleep=sleep)
                dfs.append(df)
                break

            except requests.exceptions.ConnectionError:
                time.sleep(10)
                logger.info('ERROR: caught ConnectionError. Retrying.')

    vacancy_url_df = pd.concat(dfs)[['url', 'responses']]

    logger.info(f'SUCCESS: finished scrapping vacancies URLs for professional_roles = {professional_roles}')

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_url_df, table_name='url_vs_responses', schema='main_schema', if_table_exists='replace')
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_url_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_vacancies_info(self, urls, sleep=(1,3), postgres_creds=None, save_to_postgres=False):

    logger.info(f'SUCCESS: started scrapping vacancies info')

    dfs = []
    for url in tqdm(urls):

        # try up to 5 times:
        for attempt in range(5):
            try:
                logger.info(f'SUCCESS: started scrapping {url}')
                json = requests.get(url).json()

                df = pd.DataFrame({'id':[json['id']],
                                   'name':[json['name']],
                                   'salary':[json['salary']],
                                   'key_skills':[json['key_skills']],
                                   'experience':[json['experience']],
                                   'description':[json['description']],
                                   'schedule':[json['schedule']]})

                logger.info(f'SUCCESS: finished scrapping {url}')
                break

            except requests.exceptions.ConnectionError:
                time.sleep(10)
                logger.info('ERROR: caught ConnectionError. Retrying.')

        dfs.append(df)

        # sleep:
        min_sleep = sleep[0]
        max_sleep = sleep[1]
        time.sleep(uniform(min_sleep,max_sleep))

    logger.info(f'SUCCESS: finished scrapping vacancies info')

    # preproccess data:
    logger.info(f'SUCCESS: started preprocessing data')
    vacancy_info_df = pd.concat(dfs)

    # edit description:
    vacancy_info_df.description = vacancy_info_df.description.apply(lambda x: re.sub(re.compile('<.*?>'), '', x))

    # edit key_skills:
    vacancy_info_df.key_skills = vacancy_info_df.key_skills.apply(lambda x: ', '.join([d['name'] for d in x]))

    # edit experience:
    vacancy_info_df.experience = vacancy_info_df.experience.apply(lambda x: x['name'])

    # edit schedule:
    vacancy_info_df.schedule = vacancy_info_df.schedule.apply(lambda x: x['id'])

    # edit salary:
    vacancy_info_df['salary_from'] = vacancy_info_df.salary.apply(lambda x: x['from'] if type(x)==dict else None)
    vacancy_info_df['salary_to'] = vacancy_info_df.salary.apply(lambda x: x['to'] if type(x)==dict else None)
    vacancy_info_df['salary_currency'] = vacancy_info_df.salary.apply(lambda x: x['currency'] if type(x)==dict else None)

    # choose columns:
    vacancy_info_df = vacancy_info_df[['id', 'name', 'key_skills', 'experience', 'description', 'schedule', 'salary_from', 'salary_to', 'salary_currency']]

    logger.info(f'SUCCESS: finished preprocessing data')

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_info_df, table_name='vacancy_info', schema='main_schema', if_table_exists='replace')
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_info_df