# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/04-hh_parser.ipynb (unless otherwise specified).

__all__ = ['TextPreprocessor', 'preprocess_string', 'preprocess_series', 'HeadHunterScrapper',
           'get_urls_from_professional_role', 'get_url_vs_responses', 'get_url_vs_info',
           'get_professionalroles_id_vs_name', 'SimilarityFinder', 'get_url_vs_similarity', 'Telegram', 'send_vacancy',
           'send_recommended_vacancies_to_chatid', 'send_recommended_vacancies_to_chatids_with_status_1']

# Cell

import pandas as pd
from random import choice, uniform
import requests
import numpy as np
import fastcore.basics as fcb
from .core import logger
from .postgres import PostgresConnector
from tqdm import tqdm
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics.pairwise import cosine_similarity
import time
from pymystem3 import Mystem

from .core import encode, decode
from .texts import welcome_text, after_rocket_text, settings_text, start_search_text, stop_search_text, no_recommended_vacancies_text
import telebot
from telebot.types import InlineKeyboardMarkup, InlineKeyboardButton
from time import sleep
from random import uniform

# Cell

class TextPreprocessor:
    def __init__(self):

        # get mystem object:
        self.mystem = Mystem()

# Cell

@fcb.patch_to(TextPreprocessor)

def preprocess_string(self, string):

    """
    FUNC TO PREPROCESS STRING

    Return: string

    """

    # lower text:
    string = string.lower()

    # replace ",", ".", ":", ";", "-", ", ', `  with gaps:
    string = re.sub(r'[,.:;\'\-"`]', ' ', string)

    # replace "c++" and "c#" in order not to be dropped:
    string = re.sub(r'(c\+\+|с\+\+)', 'cplusplus ', string)
    string = re.sub(r'(c#|с#)', 'csharp ', string)

    # remove all punctuation:
    string = re.sub(r'[^\w\s]', '', string)

    # remove digits if surrounded by spaces:
    string = re.sub(r'(?<=\s)\d+(?=\s+)', '', string)


    # lemmatize:
    mystem = self.mystem
    tokens = mystem.lemmatize(string)
    tokens = [token for token in tokens if token != " " and token != "\n"]
    string = " ".join(tokens)

    # remove duplicated gaps:
    string = ' '.join(string.split())

    return string

# Cell

@fcb.patch_to(TextPreprocessor)

def preprocess_series(self, series):

    """
    FUNC TO PREPROCESS Series

    Return: pd.Series

    """

    return series.apply(lambda x: self.preprocess_string(x))

# Cell

class HeadHunterScrapper:
    def __init__(self, base_url='https://api.hh.ru/vacancies'):

        self.base_url = base_url

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_urls_from_professional_role(self, professional_role, per_page=100, pages=20, sleep=(1,3)):


    """
    FUNC TO GET professional_role's vacancies URLS, RESPONSES

    Return: pd.DataFrame (columns: professional_role, url, responses, id)
    """



    logger.info(f'SUCCESS: started scrapping vacancies URLs for professional_role = {professional_role}')

    # get API's base url:
    base_url = self.base_url

    # get vacancies' urls of entered professional_role:
    urls = []
    responses = []
    ids = []
    for page in range(pages):

        logger.info(f'SUCCESS: started scrapping page {page}')

        # make request:
        params = {'professional_role':professional_role, 'area':1, 'per_page':per_page, 'page':page, 'responses_count_enabled':True} # Moscow
        # json = requests.get(base_url, params=params, proxies=proxies).json()
        json = requests.get(base_url, params=params).json()

        # break if there're no more vacancies of professional_role:
        if len(json['items'])==0:
            break

        # get list of vacancies' urls and append it:
        vacancy_urls      = pd.Series(json['items']).apply(lambda x: x['url']).to_list()
        vacancy_responses = pd.Series(json['items']).apply(lambda x: x['counters']['responses']).to_list() # кол-во откликов
        vacancy_ids       = pd.Series(json['items']).apply(lambda x: x['id']).to_list()

        urls      += vacancy_urls
        responses += vacancy_responses
        ids       += vacancy_ids

        # sleep:
        min_sleep = sleep[0]
        max_sleep = sleep[1]
        time.sleep(uniform(min_sleep,max_sleep))

        logger.info(f'SUCCESS: finished scrapping page {page}')

    # create dataframe:
    vacancy_url_df = pd.DataFrame({'professional_role':professional_role, 'url':urls, 'responses':responses, 'id':ids})

    logger.info(f'SUCCESS: finished scrapping vacancies URLs for professional_role = {professional_role}')

    return vacancy_url_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_url_vs_responses(self, professional_roles, per_page=100, pages=20, sleep=(1,3), postgres_creds=None, save_to_postgres=False):


    """
    FUNC TO GET professional_roles's vacancies URLS, RESPONSES

    Return: pd.DataFrame (columns: url, responses)
    """

    logger.info(f"SUCCESS: started scrapping urls' responses")

    dfs = []
    for professional_role in professional_roles:

        # try up to 5 times:
        for attempt in range(5):
            try:
                df = self.get_urls_from_professional_role(professional_role, per_page=per_page, pages=pages, sleep=sleep)
                dfs.append(df)
                break

            except:
                time.sleep(10)
                logger.info('ERROR. Retrying.')

    vacancy_url_df = pd.concat(dfs)[['url', 'responses', 'professional_role']]

    logger.info(f"SUCCESS: finished scrapping urls' responses")

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_url_df, table_name='url_vs_responses', schema='main_schema', if_table_exists='append') # append new rows
        pc.execute('DELETE FROM main_schema.url_vs_responses WHERE inserted_into_db_msk != (SELECT max(inserted_into_db_msk) FROM main_schema.url_vs_responses);') # drop old data
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_url_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_url_vs_info(self, urls, sleep=(1,3), postgres_creds=None, save_to_postgres=False):


    """
    FUNC TO GET vacancy's INFO

    Parameters:
    urls:list - vacancies' urls
    sleep:tuple - sleep seconds between iterations.
    postgres_creds: dict - Postgres credentials
    save_to_postgres: bool - whether to save to Postgres table "main_schema.url_vs_info" or not.

    Return: pd.DataFrame
    """

    logger.info(f"SUCCESS: started scrapping urls' info")

    dfs = []
    for url in urls:

        # try up to 5 times:
        for attempt in range(5):
            try:
                json = requests.get(url).json()

                df = pd.DataFrame({'id':[json['id']],
                                   'name':[json['name']],
                                   'salary':[json['salary']],
                                   'key_skills':[json['key_skills']],
                                   'experience':[json['experience']],
                                   'description':[json['description']],
                                   'schedule':[json['schedule']],
                                   'company':json['employer']['name']})

                break

            except:
                time.sleep(10)
                logger.info(f'ERROR while scrapping {url}. Retrying.')

        dfs.append(df)

        # sleep:
        min_sleep = sleep[0]
        max_sleep = sleep[1]
        time.sleep(uniform(min_sleep,max_sleep))

    logger.info(f"SUCCESS: finished scrapping urls' info")

    # preproccess data:
    logger.info(f'SUCCESS: started preprocessing data')
    vacancy_info_df = pd.concat(dfs)

    # remove HTML tags from description:
    vacancy_info_df.description = vacancy_info_df.description.apply(lambda x: re.sub(re.compile('<.*?>'), '', x))

    # edit key_skills:
    vacancy_info_df.key_skills = vacancy_info_df.key_skills.apply(lambda x: ', '.join([d['name'] for d in x]))

    # edit experience:
    vacancy_info_df.experience = vacancy_info_df.experience.apply(lambda x: x['name'])

    # edit schedule:
    vacancy_info_df.schedule = vacancy_info_df.schedule.apply(lambda x: x['id'])

    # edit salary:
    vacancy_info_df['salary_from'] = vacancy_info_df.salary.apply(lambda x: x['from'] if type(x)==dict else None)
    vacancy_info_df['salary_to'] = vacancy_info_df.salary.apply(lambda x: x['to'] if type(x)==dict else None)
    vacancy_info_df['salary_currency'] = vacancy_info_df.salary.apply(lambda x: x['currency'] if type(x)==dict else None)

    # get url:
    vacancy_info_df['url'] = vacancy_info_df['id'].apply(lambda x: f'https://api.hh.ru/vacancies/{x}?host=hh.ru')

    # choose columns:
    vacancy_info_df = vacancy_info_df[['id', 'url', 'name', 'company', 'key_skills', 'experience', 'description', 'schedule', 'salary_from', 'salary_to', 'salary_currency']]

    # filter currency RUR, NaN:
    vacancy_info_df = vacancy_info_df[vacancy_info_df.salary_currency.isin([None, 'RUR'])]

    # preprocess description:
    tp = TextPreprocessor()
    vacancy_info_df['description'] = tp.preprocess_series(vacancy_info_df['description'])

    logger.info(f'SUCCESS: finished preprocessing data')

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=vacancy_info_df, table_name='url_vs_info', schema='main_schema', if_table_exists='append') # append new rows
        pc.execute('DELETE FROM main_schema.url_vs_info WHERE inserted_into_db_msk != (SELECT max(inserted_into_db_msk) FROM main_schema.url_vs_info);') # drop old data
        logger.info(f'SUCCESS: finished saving to Postgres')

    return vacancy_info_df

# Cell

@fcb.patch_to(HeadHunterScrapper)

def get_professionalroles_id_vs_name(self, postgres_creds=None, save_to_postgres=False):

    """
    FUNC TO GET PROFESSIONAL ROLES: id VS. name

    Parameters:
    postgres_creds: dict - Postgres credentials
    save_to_postgres: bool - whether to save to Postgres table "main_schema.url_vs_info" or not.

    Return: pd.DataFrame
    """

    url = 'https://api.hh.ru/professional_roles'
    json = requests.get(url).json()

    ids = []
    names = []

    categories = json['categories']

    for category in categories:
        for role in category['roles']:

            id = role['id']
            name = role['name']

            names.append(name)
            ids.append(id)

    df = pd.DataFrame({'id':ids, 'name':names}).drop_duplicates(subset=['id'])

    if save_to_postgres:
        logger.info(f'SUCCESS: started saving to Postgres')
        pc = PostgresConnector(**postgres_creds)
        pc.save(df=df, table_name='professional_roles', schema='main_schema', if_table_exists='replace')
        logger.info(f'SUCCESS: finished saving to Postgres')

    return df

# Cell

class SimilarityFinder:
    def __init__(self, cv, postgres_creds, min_df=0.002, n_components=300, ngram_range=(1,1)):

        # get cv:
        self.cv = cv

        # get postgres_creds:
        self.postgres_creds = postgres_creds

        # get ngram_range:
        self.ngram_range = ngram_range

        # get min_df:
        self.min_df = min_df

        # get n_components:
        self.n_components = n_components

# Cell

@fcb.patch_to(SimilarityFinder)

def get_url_vs_similarity(self):

    # get russian stop words:
    pc = PostgresConnector(**self.postgres_creds)
    self.russian_stop_words = pc.download('SELECT * from main_schema.stop_words')['word'].to_list()
    logger.info(f"SUCCESS: extracted {len(self.russian_stop_words)} russian stop words")

    # get vacancy_info_df:
    t1 = time.time()
    vacancy_info_df = pc.download('SELECT * FROM main_schema.vacancies')
    t2 = time.time()
    logger.info(f"SUCCESS: finished downloading vacancy_info_df. Time: {t2-t1:.1f} s")

    # TfidfVectorizer:
    t1 = time.time()
    vectorizer = TfidfVectorizer(min_df=self.min_df, stop_words=self.russian_stop_words, ngram_range=self.ngram_range)
    vacancy_tfidf = vectorizer.fit_transform(vacancy_info_df['description'])
    cv_tfidf = vectorizer.transform([self.cv])
    t2 = time.time()
    logger.info(f"SUCCESS: finished TfidfVectorizer. {len(vectorizer.get_feature_names())} words. Time: {t2-t1:.1f} s")

    # LSA:
    logger.info(f"SUCCESS: started LSA")
    t1 = time.time()
    lsa = TruncatedSVD(n_components=self.n_components, random_state=1)
    vacancy_lsa = lsa.fit_transform(vacancy_tfidf)
    cv_lsa = lsa.transform(cv_tfidf)
    t2 = time.time()
    logger.info(f"SUCCESS: finished LSA. Time: {t2-t1:.1f} s")

    # calculate cosine similarity between the cv and each vacancy description:
    logger.info(f"SUCCESS: started calculating cosine similarity")
    similarity_scores = cosine_similarity(vacancy_lsa, cv_lsa)
    logger.info(f"SUCCESS: finished calculating cosine similarity")


    # Add the similarity_scores column:
    vacancy_info_df['similarity'] = similarity_scores.flatten()
    vacancy_info_df['similarity'] = vacancy_info_df['similarity'].apply(lambda x: 0 if x<0 else x)

    # add column url:
    col = vacancy_info_df['id'].apply(lambda x: f'https://hh.ru/vacancy/{x}')
    vacancy_info_df.insert(0, 'url', col)
    vacancy_info_df = vacancy_info_df.drop(['id'], axis=1) # delete column id

    return vacancy_info_df

# Cell

class Telegram:
    def __init__(self, postgres_creds, token="6137609275:AAEJcRfFHrvHW792-gXA_kCDC-d2meVE0YM"):
        self.bot = telebot.TeleBot(token=token)
        self.postgres_creds = postgres_creds
        self.pc = PostgresConnector(**postgres_creds)
        self.chatid_vs_settings = self.pc.download(f"SELECT * FROM main_schema.chatid_vs_settings")

# Cell

@fcb.patch_to(Telegram)

def send_vacancy(
    self,
    chat_id,
    name,
    company,
    url,
    experience,
    salary_from,
    salary_to,
    schedule,
    key_skills,
    similarity,
    responses
):

    """
    FUNC TO TRY TO SEND MESSAGE WITH RECOMMENDED VACANCY (IT CAN'T BE SENT IF CHAT IS DELETED)
    """

    try:
        # get salary string:
        if np.isnan(salary_from) and np.isnan(salary_to):
            salary = 'не указан'

        elif not np.isnan(salary_from) and not np.isnan(salary_to):
            salary = f'{salary_from/10**3:.0f} — {salary_to/10**3:.0f} тыс. руб.'

        elif np.isnan(salary_from) and not np.isnan(salary_to):
            salary = f'до {salary_to/10**3:.0f} тыс. руб.'

        elif not np.isnan(salary_from) and np.isnan(salary_to):
            salary = f'от {salary_from/10**3:.0f} тыс. руб.'

        # get is_remote string:
        if schedule=='remote': is_remote = 'да'
        else: is_remote = 'нет'

        # get key_skills string:
        if key_skills=='': key_skills='не указано'

        # get color circle:
        if similarity < 25: circle = '🔴'
        elif similarity>=25 and similarity<50: circle = '🟠'
        elif similarity >= 50 and similarity < 75: circle = '🟡'
        else: circle = '🟢'

        # get message text:
        text = f"""
<a href="{url}"><b>{name}</b></a>

<b>Компания:</b> <i>{company}</i>

<b>Требуемый опыт:</b> <i>{experience}</i>

<b>Уровень дохода:</b> <i>{salary}</i>

<b>Ключевые навыки:</b> <i>{key_skills}</i>

<b>Удаленный формат:</b> <i>{is_remote}</i>

<b>👥 Откликов:</b> <i>{responses}</i>
<b>{circle} Соответствие резюме:</b> <i>{similarity} %</i>
"""

        keyboard = InlineKeyboardMarkup()
        button1   =  InlineKeyboardButton(text="🔍 Перейти на страницу вакансии", url=url)
        keyboard.add(button1)

        button2   =  InlineKeyboardButton(text="⚙️ Настроить поиск", callback_data="settings")
        keyboard.add(button2)

        button3   =  InlineKeyboardButton(text="❌ Удалить", callback_data="delete_vacancy")
        keyboard.add(button3)

        button4 = InlineKeyboardButton(text="➡️ Показать еще", callback_data="show_more")
        keyboard.add(button4)

        self.bot.send_message(chat_id, text, parse_mode="HTML", reply_markup=keyboard)

        # fictive message to create extra message_id after message with vacancy:
        m = self.bot.send_message(chat_id, text='⏳ Загрузка...')
        self.bot.delete_message(chat_id, m.message_id)
    except:
        pass

# Cell

@fcb.patch_to(Telegram)

def send_recommended_vacancies_to_chatid(self, chat_id):

    """
    FUNC TO SEND TOP 10 RECOMMENDED VACANCIES TO USER WITH chat_id

    Parameters:
    chat_id: int

    """

    chatid_vs_settings = self.chatid_vs_settings

    # get settings:
    settings = chatid_vs_settings[chatid_vs_settings['chat_id']==chat_id]
    chat_id = settings['chat_id'].values[0]
    cv = settings['cv'].values[0]
    max_responses = settings['max_responses'].values[0]
    min_salary = settings['min_salary'].values[0]
    experience = settings['experience'].values[0]

    # get url_vs_similarity:
    sf = SimilarityFinder(cv, self.postgres_creds)
    url_vs_similarity = sf.get_url_vs_similarity()

    # get filters:
    if max_responses==None:
        max_responses_filter = (url_vs_similarity['responses']<100_000)
    else:
        max_responses_filter = (url_vs_similarity['responses']<int(max_responses))

    if min_salary==None:
        min_salary_filter = True
    else:
        min_salary_filter = (url_vs_similarity['salary_from']>int(min_salary))

    if experience==None:
        experience_filter = True
    else:
        experience_filter = (url_vs_similarity['experience'].isin(experience.split(', ')))

    # get sent vacancies urls:
    sent_vacancies_urls = self.pc.download(f'SELECT url FROM main_schema.sent_vacancies WHERE chat_id = {chat_id}')['url'].to_list()

    # filter vacancies:
    filtered_vacancies = url_vs_similarity[max_responses_filter & min_salary_filter & experience_filter].sort_values('similarity', ascending=False)
    filtered_vacancies = filtered_vacancies[~filtered_vacancies['url'].isin(sent_vacancies_urls)] # exclude vacancies already sent to this chat_id

    # send top 10 vacancies to chat_id:
    if filtered_vacancies.shape[0] > 0:
        for _, row in filtered_vacancies.head(10).iterrows():
            url = row['url']
            name = row['name']
            company = row['company']
            key_skills = ', '.join(row['key_skills'].split(', ')[:4:]) # get first 5 key skills
            experience = row['experience']
            schedule = row['schedule']
            salary_from = row['salary_from']
            salary_to = row['salary_to']
            responses = row['responses']
            similarity = round(np.sqrt(row['similarity'])*100, 1) # to look more attractive :)

            # try to send recommended vacancy:
            self.send_vacancy(chat_id, name, company, url, experience, salary_from, salary_to, schedule, key_skills, similarity, responses)

            # save to sent_vacancies table:
            self.pc.save(df=pd.DataFrame({"chat_id":[chat_id], 'url':[url]}), table_name='sent_vacancies', schema='main_schema')
            sleep(uniform(1, 1.2))

    else:
        self.bot.send_message(chat_id, no_recommended_vacancies_text())


# Cell

@fcb.patch_to(Telegram)

def send_recommended_vacancies_to_chatids_with_status_1(self):

    """
    FUNC TO SEND TOP 10 RECOMMENDED VACANCIES TO USERS WITH STATUS = 1
    """

    # get chat_ids with status=1:
    chat_ids = pc.download('SELECT chat_id FROM main_schema.chatid_vs_status WHERE status=1')['chat_id'].to_list()

    # send recommendation to selected users:
    for chat_id in chat_ids:
        self.send_recommended_vacancies_to_chatid(chat_id)