# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/04-hh_parser.ipynb (unless otherwise specified).

__all__ = ['TextPreprocessor', 'preprocess_string', 'preprocess_series', 'HeadHunterScrapper', 'get_url_vs_responses',
           'get_url_vs_info', 'SimilarityFinder', 'get_url_vs_similarity']

# Cell

import pandas as pd
from random import choice, uniform
import requests
import numpy as np
import fastcore.basics as fcb
from .core import logger
from .postgres import PostgresConnector
from tqdm import tqdm
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics.pairwise import cosine_similarity
import time
from pymystem3 import Mystem

# Cell

class TextPreprocessor:
    def __init__(self):

        # get mystem object:
        self.mystem = Mystem()

# Cell

@fcb.patch_to(TextPreprocessor)

def preprocess_string(self, string):

    """
    FUNC TO PREPROCESS STRING

    Return: string

    """

    # lower text:
    string = string.lower()

    # replace ",", ".", ":", ";", "-", ", ', `  with gaps:
    string = re.sub(r'[,.:;\'\-"`]', ' ', string)

    # replace "c++" and "c#" in order not to be dropped:
    string = re.sub(r'(c\+\+|с\+\+)', 'cplusplus ', string)
    string = re.sub(r'(c#|с#)', 'csharp ', string)

    # remove all punctuation:
    string = re.sub(r'[^\w\s]', '', string)

    # remove digits if surrounded by spaces:
    string = re.sub(r'(?<=\s)\d+(?=\s+)', '', string)


    # lemmatize:
    mystem = self.mystem
    tokens = mystem.lemmatize(string)
    tokens = [token for token in tokens if token != " " and token != "\n"]
    string = " ".join(tokens)

    # remove duplicated gaps:
    string = ' '.join(string.split())

    return string

# Cell

@fcb.patch_to(TextPreprocessor)

def preprocess_series(self, series):

    """
    FUNC TO PREPROCESS Series

    Return: pd.Series

    """

    return series.apply(lambda x: self.preprocess_string(x))

# Cell

class HeadHunterScrapper:
    def __init__(self, base_url='https://api.hh.ru/vacancies'):

        self.base_url = base_url

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_url_vs_responses(self, professional_roles, per_page=100, pages=20, sleep=(1,3), postgres_creds=None, save_to_postgres=False):


    """
    FUNC TO GET professional_roles's vacancies URLS, RESPONSES

    Return: pd.DataFrame (columns: url, responses)
    """

    logger.info(f"SUCCESS: started scrapping urls' responses")

    dfs = []
    for professional_role in professional_roles:

        # try up to 5 times:
        for attempt in range(5):
            try:
                df = self.get_urls_from_professional_role(professional_role, per_page=per_page, pages=pages, sleep=sleep)
                dfs.append(df)
                break

            except:
                time.sleep(10)
                logger.info('ERROR. Retrying.')

    vacancy_url_df = pd.concat(dfs)[['url', 'responses']]

    logger.info(f"SUCCESS: finished scrapping urls' responses")

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_url_df, table_name='url_vs_responses', schema='main_schema', if_table_exists='append') # append new rows
        pc.execute('DELETE FROM main_schema.url_vs_responses WHERE inserted_into_db_msk != (SELECT max(inserted_into_db_msk) FROM main_schema.url_vs_responses);') # drop old data
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_url_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_url_vs_info(self, urls, sleep=(1,3), postgres_creds=None, save_to_postgres=False):


    """
    FUNC TO GET vacancy's INFO

    Return: pd.DataFrame
    """

    logger.info(f"SUCCESS: started scrapping urls' info")

    dfs = []
    for url in urls:

        # try up to 5 times:
        for attempt in range(5):
            try:
                json = requests.get(url).json()

                df = pd.DataFrame({'id':[json['id']],
                                   'name':[json['name']],
                                   'salary':[json['salary']],
                                   'key_skills':[json['key_skills']],
                                   'experience':[json['experience']],
                                   'description':[json['description']],
                                   'schedule':[json['schedule']],
                                   'company':json['employer']['name']})

                break

            except:
                time.sleep(10)
                logger.info(f'ERROR while scrapping {url}. Retrying.')

        dfs.append(df)

        # sleep:
        min_sleep = sleep[0]
        max_sleep = sleep[1]
        time.sleep(uniform(min_sleep,max_sleep))

    logger.info(f"SUCCESS: finished scrapping urls' info")

    # preproccess data:
    logger.info(f'SUCCESS: started preprocessing data')
    vacancy_info_df = pd.concat(dfs)

    # remove HTML tags from description:
    vacancy_info_df.description = vacancy_info_df.description.apply(lambda x: re.sub(re.compile('<.*?>'), '', x))

    # edit key_skills:
    vacancy_info_df.key_skills = vacancy_info_df.key_skills.apply(lambda x: ', '.join([d['name'] for d in x]))

    # edit experience:
    vacancy_info_df.experience = vacancy_info_df.experience.apply(lambda x: x['name'])

    # edit schedule:
    vacancy_info_df.schedule = vacancy_info_df.schedule.apply(lambda x: x['id'])

    # edit salary:
    vacancy_info_df['salary_from'] = vacancy_info_df.salary.apply(lambda x: x['from'] if type(x)==dict else None)
    vacancy_info_df['salary_to'] = vacancy_info_df.salary.apply(lambda x: x['to'] if type(x)==dict else None)
    vacancy_info_df['salary_currency'] = vacancy_info_df.salary.apply(lambda x: x['currency'] if type(x)==dict else None)

    # get url:
    vacancy_info_df['url'] = vacancy_info_df['id'].apply(lambda x: f'https://api.hh.ru/vacancies/{x}?host=hh.ru')

    # choose columns:
    vacancy_info_df = vacancy_info_df[['id', 'url', 'name', 'company', 'key_skills', 'experience', 'description', 'schedule', 'salary_from', 'salary_to', 'salary_currency']]

    # filter currency RUR, NaN:
    vacancy_info_df = vacancy_info_df[vacancy_info_df.salary_currency.isin([None, 'RUR'])]

    # preprocess description:
    tp = TextPreprocessor()
    vacancy_info_df['description'] = tp.preprocess_series(vacancy_info_df['description'])

    logger.info(f'SUCCESS: finished preprocessing data')

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_info_df, table_name='url_vs_info', schema='main_schema', if_table_exists='append') # append new rows
        pc.execute('DELETE FROM main_schema.url_vs_info WHERE inserted_into_db_msk != (SELECT max(inserted_into_db_msk) FROM main_schema.url_vs_info);') # drop old data
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_info_df

# Cell

class SimilarityFinder:
    def __init__(self, cv, postgres_creds, min_df=0.002, n_components=300, ngram_range=(1,1)):

        # get cv:
        self.cv = cv

        # get postgres_creds:
        self.postgres_creds = postgres_creds

        # get ngram_range:
        self.ngram_range = ngram_range

        # get min_df:
        self.min_df = min_df

        # get n_components:
        self.n_components = n_components

# Cell

@fcb.patch_to(SimilarityFinder)

def get_url_vs_similarity(self):

    # get russian stop words:
    pc = PostgresConnector(**self.postgres_creds)
    self.russian_stop_words = pc.download('SELECT * from main_schema.stop_words')['word'].to_list()
    logger.info(f"SUCCESS: extracted {len(self.russian_stop_words)} russian stop words")

    # get vacancy_info_df:
    t1 = time.time()
    vacancy_info_df = pc.download('SELECT * FROM main_schema.vacancies')
    t2 = time.time()
    logger.info(f"SUCCESS: finished downloading vacancy_info_df. Time: {t2-t1:.1f} s")

    # TfidfVectorizer:
    t1 = time.time()
    vectorizer = TfidfVectorizer(min_df=self.min_df, stop_words=self.russian_stop_words, ngram_range=self.ngram_range)
    vacancy_tfidf = vectorizer.fit_transform(vacancy_info_df['description'])
    cv_tfidf = vectorizer.transform([self.cv])
    t2 = time.time()
    logger.info(f"SUCCESS: finished TfidfVectorizer. {len(vectorizer.get_feature_names())} words. Time: {t2-t1:.1f} s")

    # LSA:
    t1 = time.time()
    lsa = TruncatedSVD(n_components=self.n_components, random_state=1)
    vacancy_lsa = lsa.fit_transform(vacancy_tfidf)
    cv_lsa = lsa.transform(cv_tfidf)
    t2 = time.time()
    logger.info(f"SUCCESS: finished LSA. Time: {t2-t1:.1f} s")

    # calculate cosine similarity between the cv and each vacancy description:
    similarity_scores = cosine_similarity(vacancy_lsa, cv_lsa)
    logger.info(f"SUCCESS: finished calculating cosine similarity")


    # Add the similarity_scores column:
    vacancy_info_df['similarity'] = similarity_scores.flatten()
    vacancy_info_df['similarity'] = vacancy_info_df['similarity'].apply(lambda x: 0 if x<0 else x)

    # add column url:
    col = vacancy_info_df['id'].apply(lambda x: f'https://hh.ru/vacancy/{x}')
    vacancy_info_df.insert(0, 'url', col)
    vacancy_info_df = vacancy_info_df.drop(['id'], axis=1) # delete column id

    return vacancy_info_df