# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/05-preprocessor.ipynb (unless otherwise specified).

__all__ = ['Preprocessor', 'preprocess_description', 'run']

# Cell

import pandas as pd
import fastcore.basics as fcb
from .core import logger
from pymystem3 import Mystem
import re

# Cell

class Preprocessor:
    def __init__(self):

        # get mystem object:
        self.mystem = Mystem()

# Cell

@fcb.patch_to(Preprocessor)

def preprocess_description(self, vacancy_df):

    """
    FUNC TO PREPROCESS VACANCIES (CLEAN PUNCTUATION, CLEAN DIGITS AND LEMMATIZE)

    RETURN: pd.DataFrame

    """

    # lower text:
    vacancy_df['description'] = vacancy_df['description'].str.lower()

    # replace ",", ".", ":", ";", "-" with gaps:
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: re.sub(r'[,.:;-]', ' ', x))

    # replace "c++" and "c#" in order not to be dropped:
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: re.sub(r'(c\+\+|с\+\+)', 'cplusplus ', x))
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: re.sub(r'(c#|с#)', 'csharp ', x))

    # remove all punctuation:
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: re.sub(r'[^\w\s]', '', x))

    # remove digits if surrounded by spaces:
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: re.sub(r'(?<=\s)\d+(?=\s)', '', x))

    # lemmatize:
    def lemmatize(text):
        mystem = self.mystem
        tokens = mystem.lemmatize(text)
        tokens = [token for token in tokens if token != " " and token != "\n"]
        text = " ".join(tokens)
        return text

    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: lemmatize(x))

    # remove duplicated gaps:
    vacancy_df['description'] = vacancy_df['description'].apply(lambda x: ' '.join(x.split()))

    return vacancy_df

# Cell

@fcb.patch_to(Preprocessor)

def run(self, vacancy_df):

    """
    FUNC TO RUN PREPROCESSING

    RETURN: pd.DataFrame

    """

    logger.info('SUCCESS. started preprocessing vacancies')


    # get vacancy_df:
    vacancy_df['description'] = vacancy_df['description'] + vacancy_df['name']

    # preprocess descriptions:
    vacancy_df = self.preprocess_description(vacancy_df)

    # filter currency:
    vacancy_df = vacancy_df[vacancy_df.salary_currency.isin([None, 'RUR'])]

    logger.info('SUCCESS. finished preprocessing vacancies')

    return vacancy_df