"""This module computes similarity values between 0 and 1 for a set of candidate pairs
for each configured property of the original datasets.
A similarity value of 1 means that the two property values are identical
while a similarity value of 0 means that the two property values are completely different.
The module differs between the following categories of similarity functions which allow
different strategies to compute the similarity values for all candidate pairs efficiently:

- cosine similarity between the embedding vectors corresponding to two strings;
the embedding vectors are generated by means of a trained language model

- cosine similarity between the bag of words corresponding to two strings;
the words in the bag have TFIDF scores as weights

- similarity functions between numerical values

- the "equal" function that returns 1 for identity and 0 otherwise

- further similarity functions that are not yet implemented in an efficient way
"""
import collections
import logging
import time
from typing import Any, Callable, List, Optional, Set, Tuple, cast

import numpy as np
import pandas as pd
import scipy.sparse
import sentence_transformers
import sentence_transformers.util
from sklearn.feature_extraction.text import TfidfVectorizer
from thefuzz import fuzz
from tqdm import tqdm

import matchain.util


def get_similarity_function_names() -> List[str]:
    """
    Returns the names of all implemented similarity functions.

    :return: list of similarity function names
    :rtype: List[str]
    """
    return [
        'absolute', 'relative', 'equal', 'tfidf', 'embedding', 'fuzzy',
        'tfidf_sklearn',
        'shingle_tfidf', # synonym for tfidf_sklearn
    ]


def create_property_mapping(params_mapping: dict,
                            maxidf: int = -1) -> List[dict]:
    """Creates a property mapping for a given params mapping.
    A property mapping is a list of dicts with the following keys:
    - column_pos: the position of the column in the dataframe
    - column_name: the name of the column in the dataframe
    - prop1: the name of the first property
    - prop2: the name of the second property
    - sim_fct_name: the name of the similarity function
    - sim_fct_params: the parameters of the similarity function

    :param params_mapping: the params mapping as defined in the config file
    :type params_mapping: dict
    :param maxidf: the maximum inverse document frequency
    :type maxidf: int
    :return: the property mapping
    :rtype: List[dict]
    """
    property_mapping = []
    i = -1
    for prop, sim_fcts in params_mapping.items():
        if isinstance(sim_fcts, str):
            sim_fcts = [sim_fcts]
        for fct in sim_fcts:
            if fct == 'shingle_tfidf':
                fct = 'tfidf_sklearn'
            i += 1
            params_sim_fct = {}
            if fct == 'absolute':
                params_sim_fct = {'cut_off_value': 10}
            elif fct == 'tfidf':
                params_sim_fct = {'maxidf': maxidf}

            propmap = {
                'column_pos': i,
                'column_name': str(i),
                'prop1': prop,
                'prop2': prop,
                'sim_fct_name': fct,
                'sim_fct_params': params_sim_fct
            }
            property_mapping.append(propmap)

    return property_mapping


def get_tfidf_props(params_mapping: dict) -> List[str]:
    """Filters the properties that are used for tfidf similarity.

    :param params_mapping: the params mapping as defined in the config file
    :type params_mapping: dict
    :return: the list of properties that are used for tfidf similarity
    :rtype: List[str]
    """
    props = set()
    for prop, sim_fcts in params_mapping.items():
        if isinstance(sim_fcts, str):
            sim_fcts = [sim_fcts]
        if 'tfidf' in sim_fcts:
            props.add(prop)
    return list(props)


class PseudoVectorizedSimilarity():
    """A wrapper to compute similarity values for similarity functions
    that are not computed in a vectorized manner.
    """

    @staticmethod
    def _fuzzy(v: str, w: str):
        value = fuzz.token_set_ratio(v, w)
        return value / 100

    @staticmethod
    def compute_similarity(property_mapping: List[dict],
                           lifted_array: np.ndarray,
                           df_sim: pd.DataFrame) -> pd.DataFrame:
        """Uses numpy's vectorize method to compute similarity values. Please note:
        'The function np.vectorize is provided primarily for convenience,
        not for performance. The implementation is essentially a for loop.'
        (see https://numpy.org/doc/stable/reference/generated/numpy.vectorize.html#numpy-vectorize)

        :param property_mapping: Describes the similarity functions to be used
        :type property_mapping: List[dict]
        :param lifted_array: see AdvancedIndexHelper.lift_index_to_candidates
        :type lifted_array: np.ndarray
        :param df_sim: the dataframe to which the similarity values are added
        :type df_sim: pd.DataFrame
        :raises ValueError: if the similarity function is not implemented
        :return: the dataframe with the similarity values added
        :rtype: pd.DataFrame
        """

        for i_prop, propmap in enumerate(property_mapping):
            sim_fct_name = propmap['sim_fct_name']
            if sim_fct_name == 'fuzzy':
                sim_fct = PseudoVectorizedSimilarity._fuzzy
            else:
                raise ValueError(f'Unknown similarity function={sim_fct_name}')

            vect_fct = np.vectorize(sim_fct)
            v = lifted_array[:, 2 * i_prop]
            w = lifted_array[:, 2 * i_prop + 1]
            result = vect_fct(v, w)

            col = propmap['column_name']
            df_sim[col] = result

        return df_sim


class VectorizedSimilarity():
    """Defines some similarity functions in a vectorized manner,
    i.e. similarity function can be applied to pairs of numpy arrays
    instead of pairs to values very efficiently.
    """

    @staticmethod
    def _dist_absolute(v, w):
        result = np.abs(v - w)
        result = result.astype(float)
        return result

    @staticmethod
    def _dist_relative(v, w):
        nom = abs(v - w)
        den = np.maximum(np.abs(v), np.abs(w))
        result = nom / den
        #result = np.where(den > 0, nom / den, 0)
        result = result.astype(float)
        return result

    @staticmethod
    def _dist_equal(v, w):
        result = np.where(v != w, 1, 0)
        # the previous statement results into 0 if either v[i]=np.nan or w[i]=np.nan
        # or into 1 if both v[i]=w[i]=np.nan
        # the next statement corrects this
        # use pd.isnull instead of np.isnan to avoid TypeError when comparing string
        # instead of floats
        #result = np.where(pd.isnull(v) | pd.isnull(w), np.nan, result)
        return result

    @staticmethod
    def _convert_to_similarity_fct(
            dist_fct: Callable[[Any, Any], Any],
            cut_off_value: float) -> Callable[[Any, Any], Any]:
        """Converts a distance function to a similarity function.

        :param dist_fct: a distance function
        :type dist_fct: Callable[[Any, Any], Any]
        :param cut_off_value: distance value above which the similarity is 0
        :type cut_off_value: float
        :return: _description_
        :rtype: Callable[[Any, Any], Any]
        """

        # parameters to describe a monotone decreasing conversion fct c:[0, oo) --> [0,1]
        # with c(0) = 1
        # cut_off_mode could also be 'max' or 'quantile' (with cut_off_value = percentage)
        # monotonicity could also by 1 - 1/x for (discrete values 1, 2, 3) or 1/exponential or ...
        # (fixed, 1, linear) --> score(v1, v2) = 1 - dist_fct(v1, v2)
        def convert_to_similarity_fct_internal(v, w):
            dist = dist_fct(v, w)
            sim = 1 - np.minimum(dist, cut_off_value) / cut_off_value
            return sim

        return convert_to_similarity_fct_internal

    @staticmethod
    def create_similarity_function_from_params(
            fct_name: str, cut_off_value: float) -> Callable[[Any, Any], Any]:
        """Looks up the distance function expressed
        in terms of numpy. The returned function allows for
        efficient vectorized computation of similarity values.

        :param fct_name: the name of the similarity function
        :type fct_name: str
        :param cut_off_value: distance value above which the similarity is 0
        :type cut_off_value: float
        :raises ValueError: if the distance function is not implemented
        :return: the similarity function
        :rtype: Callable[[Any, Any], Any]
        """
        if fct_name == 'absolute':
            dist_fct = VectorizedSimilarity._dist_absolute
        elif fct_name == 'relative':
            dist_fct = VectorizedSimilarity._dist_relative
        elif fct_name == 'equal':
            dist_fct = VectorizedSimilarity._dist_equal
        else:
            raise ValueError('distance function is not implemented', fct_name)

        sim_fct = VectorizedSimilarity._convert_to_similarity_fct(
            dist_fct, cut_off_value)
        return sim_fct


class ShingleWrapper:
    """Creates vectors of shingles with TF-IDF weights.
    """

    def __init__(self, shingle_size: int):
        """Init the ShingleWrapper

        :param shingle_size: size of the shingles
        :type shingle_size: int
        """
        self.shingle_size = shingle_size

    def create_shingles(self, s: str) -> Set[str]:
        """Creates the shingles for the given string.
        Example: The shingles of size 3 for string 'matching' are
        {'mat', 'atc', 'tch', 'chi', 'hin', 'ing'}.

        :param s: The string to create the shingles for.
        :type s: str
        :return: The set of shingles.
        :rtype: Set[str]
        """
        k = self.shingle_size
        return set(s[i:i + k] for i in range(len(s) - k + 1))

    def generate_vectors(self, values: pd.Series) -> scipy.sparse.csr_matrix:
        """Creates shingles from the values
        and computes the TF-IDF weights of the shingles.

        :param values: series containing the data
        :type values: pd.Series
        :return: sparse matrix with TF-IDF weights
        :rtype: scipy.sparse.csr_matrix
        """

        start_time = time.time()
        analyzer = self.create_shingles
        vectorizer = TfidfVectorizer(min_df=1, analyzer=analyzer)
        tf_idf_matrix = vectorizer.fit_transform(values)
        logging.debug('tf_idf_matrix=%s, time=%s', tf_idf_matrix.shape, time.time() - start_time)

        return cast(scipy.sparse.csr_matrix, tf_idf_matrix)


class SentenceTransformerWrapper():
    """This class wraps the SentenceTransformer library to compute
    sentence embeddings for string values and to compute the cosine similarity
    between a pair of sentence embeddings.
    """

    def __init__(self, model_name: str, device: str, batch_size: int) -> None:
        """Initializes the wrapped sentence transformer.

        :param model_name: the name of the language model used by sentence transformer,
            see https://www.sbert.net/docs/pretrained_models.html
        :type model_name: str
        :param device: the device used for computation, i.e. 'cpu' or 'cuda'
        :type device: str
        :param batch_size: the batch size used for computing embeddings
        :type batch_size: int
        """
        logging.info(
            'initializing SentenceTransformer with model=%s, device=%s, batch_size=%s',
            model_name, device, batch_size)
        self.batch_size = batch_size
        self.model = sentence_transformers.SentenceTransformer(model_name,
                                                               device=device)

    def generate_vectors(self, values: List[str]) -> np.ndarray:
        """Computes embedding vectors for the given values and
        stores them as rows in a numpy array in the order of the values.
        The vectors are normalized such that the length of each vector is 1.

        :param values: the values for which the embeddings are computed
        :type values: List[str]
        :return: the normalized embeddings as array
        :rtype: np.ndarray
        """
        logging.info('values=%s', len(values))
        embeddings = self.model.encode(values,
                                       batch_size=self.batch_size,
                                       convert_to_numpy=True,
                                       show_progress_bar=True,
                                       normalize_embeddings=True)
        return cast(np.ndarray, embeddings)


class AdvancedIndexHelper():
    """This class contains some methods that helps to compute similarity values between
    vector representations of strings efficiently."""

    @staticmethod
    def compute_cosine_similarity(
            candidate_pairs: pd.MultiIndex, index_array: np.ndarray,
            prop_columns: List[str],
            values_as_vectors: np.ndarray) -> pd.DataFrame:
        """Computes the cosine similarity between the embeddings of the
        candidate pairs for each property in prop_columns. The method
        uses numpy's advanced indexing to efficiently compute the similarity
        values.

        :param candidate_pairs: the set of candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param index_array: Each row corresponds to a row in the originally
            stacked datasets. The first column contains the embedding index
            for the first property in prop_columns, etc.
        :type index_array: np.ndarray
        :param prop_columns: the property columns
        :type prop_columns: List[str]
        :param values_as_vectors: the ndarray of vectors for values
        :type values_as_vectors: np.ndarray
        :return: dataframe where rows correspond to candidate pairs and
            columns to the similarity values for all properties
        :rtype: pd.DataFrame
        """

        lifted_index = AdvancedIndexHelper.lift_index_to_candidates(
            candidate_pairs, index_array)
        df_lifted = pd.DataFrame(data=lifted_index.copy(),
                                 index=candidate_pairs)

        for i_prop, prop_col in enumerate(prop_columns):
            col_a = 2 * i_prop
            col_b = col_a + 1
            mask = (pd.notnull(df_lifted[col_a])
                    & pd.notnull(df_lifted[col_b]))
            df_notnull = df_lifted[mask]
            logging.debug('df_lifted=%s, prop=%s, df_notnull=%s',
                          df_lifted.shape, prop_col, df_notnull.shape)

            all_dots = []
            for idx1 in tqdm(df_notnull.index.get_level_values(0).unique(),
                             desc='fastdot'):
                list_idx2 = cast(pd.MultiIndex,
                                 df_notnull.index).get_loc_level(idx1,
                                                                 level=0)[1]
                i_emb_a = index_array[idx1, i_prop]
                i_emb_b_array = index_array[list_idx2, i_prop]
                i_emb_b_array = i_emb_b_array.astype(int)
                emb_a = values_as_vectors[i_emb_a]
                emb_b_array = values_as_vectors[i_emb_b_array].T
                dot = np.dot(emb_a, emb_b_array)
                all_dots.append(dot)

            all_dots = np.hstack(all_dots)
            all_dots = np.clip(all_dots, a_min=0, a_max=1)
            df_lifted.at[df_notnull.index, prop_col] = all_dots

        return df_lifted[prop_columns]

    @staticmethod
    def create_index_array_from_property_values(
            df_data: pd.DataFrame, props: List[str],
            values: Optional[pd.Series]) -> Tuple[List[str], np.ndarray]:
        """This method collects all unique string values for the given properties
        and enumerates them starting from 0. The returned index array has dimensions
        (n, m) where n is the number of rows in the given dataframe and m is the
        number of properties. The integers in the index array are the indices of the
        string values in the list of unique string values. This method prepares the
        efficient computation of embeddings and cosine similarity values in a later step.

        :param df_data: the data of both datasets
        :type df_data: pd.DataFrame
        :param props: the properties to be used for creating embeddings
        :type props: List[str]
        :param values: values used already during blocking
        :type values: Optional[pd.Series]
        :return: the list of additional unique string values from all given properties which are
            not in the given values (which only correspond to blocking properties)
            and the index array
        :rtype: Tuple[List[str], np.ndarray]
        """
        logging.debug('start create_index_array_from_property_values')
        value2int = {}
        n_original_values = 0
        if values is not None:
            value2int = {value: i for i, value in enumerate(values)}
            n_original_values = len(value2int)

        rows_int = []
        df_tmp = df_data[props]
        for tup in df_tmp.itertuples(index=False, name=None):
            row_int = []
            for value in tup:
                if matchain.util.notnull(value):
                    i = value2int.get(value)
                    if i is None:
                        i = len(value2int)
                        value2int[value] = i
                    row_int.append(i)
                else:
                    row_int.append(None)
            rows_int.append(row_int)

        # the position of a value v in values is the integer-valued index
        new_values = list(value2int.keys())[n_original_values:]
        # len(index_array) == len(df_data)
        # index_array contains the integer-valued index corresponding to values in df_data
        # can also contain None for missing values
        index_array = np.array(rows_int)

        n_values = 0 if values is None else len(values)
        logging.debug('df_data=%s, index_array=%s, values=%s, new_values=%s',
                      len(df_data), len(index_array), n_values,
                      len(new_values))
        return new_values, index_array

    @staticmethod
    def lift_index_to_candidates(candidate_pairs: pd.MultiIndex,
                                 index_array: np.ndarray) -> np.ndarray:
        """The candidate pairs can be considered as two columns idx_1 and idx_2
        where idx_1 and idx_2 are indices into the given index_array
        (the numpy array behind the original data frame).
        For each column (property) in index_array, the method creates a column with the property
        values from index_array according to idx_1 and a column according to idx_2.
        All created columns are stacked vertically into a new array and returned.
        The dimensions of the returned array are (len(candidate_pairs), n)
        where n = 2 * number of columns in index_array.

        :param candidate_pairs: the set of candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param index_array: the numpy array behind the original data frame
        :type index_array: np.ndarray
        :return: the stacked values copied from index_array
        :rtype: np.ndarray
        """
        all_columns_candidates = []

        n_prop = index_array.shape[1]
        for iprop in range(n_prop):
            for level in range(2):
                list_idx = candidate_pairs.get_level_values(level=level)
                # apply advanced indexing
                # see https://numpy.org/doc/stable/user/basics.indexing.html#advanced-indexing
                column_candidates = index_array[list_idx, iprop]
                all_columns_candidates.append(column_candidates)

        lifted = np.vstack(all_columns_candidates).T
        logging.debug(
            'candidate_pairs=%s, index_array=%s, props=%s, lifted=%s',
            len(candidate_pairs), index_array.shape, n_prop, lifted.shape)
        return lifted


class TfidfSimilarity():
    """This class uses the token index to compute a sparse tfidf matrix
    and then computes the cosine similarity between those tfidf vectors
    (rows of the tfidf matrix) that correspond to candidate pairs."""

    @staticmethod
    def compute_similarity(prop: str, prop_column: str,
                           df_token_index: pd.DataFrame,
                           candidate_pairs: pd.MultiIndex,
                           df_sim: pd.DataFrame, size_1: int, size_2: int,
                           max_idf: int) -> None:
        """Computes a sparse tfidf matrix according to the given token index.
        and computes the cosine similarity between those tfidf vectors (rows
        of the tfidf matrix) that correspond to candidate pairs.
        The similarity values are computed by matrix multiplication of the
        tfidf matrix with its transpose.

        :param prop: the name of the property for which the similarity is computed
        :type prop: str
        :param prop_column: the internal column name of the property
        :type prop_column: str
        :param df_token_index: the token index
        :type df_token_index: pd.DataFrame
        :param candidate_pairs: the set of candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param df_sim: the dataframe to which the similarity values are added
        :type df_sim: pd.DataFrame
        :param size_1: the size of the first dataset
        :type size_1: int
        :param size_2: the size of the second dataset
        :type size_2: int
        :param max_idf: the maximum idf value
        :type max_idf: int
        """

        df_tmp = df_token_index.copy()
        df_tmp.insert(loc=1, column='tfidf', value=0.)
        data_size = size_1 + size_2

        m_tfidf, indices_with_tokens = TfidfSimilarity._create_sparse_tfidf_matrix(
            df_tmp, data_size, max_idf, prop)

        m_tfidf_norm = TfidfSimilarity._normalize_tfidf_matrix(m_tfidf)

        tfidf_sim = TfidfSimilarity.dot_for_candidate_pairs(
            candidate_pairs, m_tfidf_norm, size_1)

        #df_sim[prop_column] = tfidf_sim
        #return

        # Set the similarity scores for pairs with empty data to nan instead of 0.
        # This hack is necessary to obtain exactly the same results as before
        df_tmp = pd.DataFrame(index=candidate_pairs)
        df_tmp.index.set_names(['idx_1', 'idx_2'], inplace=True)
        matchain.util.sort_pairs(df_tmp)
        df_tmp[prop_column] = tfidf_sim
        idx_null = df_tmp.index.get_level_values(level=0).unique()
        idx_null = idx_null.difference(indices_with_tokens)
        for idx in idx_null:
            df_tmp.at[idx, prop_column] = None

        idx_null = df_tmp.index.get_level_values(level=1).unique()
        idx_null = idx_null.difference(indices_with_tokens)
        df_tmp = df_tmp.reorder_levels(['idx_2', 'idx_1'])
        for idx in idx_null:
            df_tmp.at[idx, prop_column] = None
        df_tmp = df_tmp.reorder_levels(['idx_1', 'idx_2'])

        df_sim[prop_column] = df_tmp[prop_column]

    @staticmethod
    def _create_sparse_tfidf_matrix(
            df_index_tokens: pd.DataFrame, data_size: int, n_max_idf: int,
            prop: str) -> Tuple[scipy.sparse.lil_matrix, list]:

        logging.debug('creating matrix for prop=%s', prop)

        ser_counts = df_index_tokens['count_1'] + df_index_tokens['count_2']
        mask = ser_counts == 0
        ser_counts[mask] = n_max_idf
        ser_tfidf = (n_max_idf / ser_counts).clip(lower=1)
        ser_tfidf = np.log(ser_tfidf)
        df_index_tokens['tfidf'] = ser_tfidf

        n_tokens = len(df_index_tokens)
        m_tfidf = scipy.sparse.lil_matrix((data_size, n_tokens))
        indices_with_tokens = set()
        column_links_1 = f'links_1_{prop}'
        column_links_2 = f'links_2_{prop}'
        df_tmp = df_index_tokens[['tfidf', column_links_1, column_links_2]]

        for token_int, tfidf, links_1, links_2 in df_tmp.itertuples(index=True,
                                                                    name=None):
            for links in [links_1, links_2]:
                if isinstance(links, set):
                    link_list = list(links)
                    indices_with_tokens.update(link_list)
                    m_tfidf[link_list, token_int] = tfidf

        #delta = 0.01
        #logging.debug(
        #    'created matrix for prop=%s, m_tfidf=%s, indices_with_tokens=%s' +
        #    ', all tfidf weights=%s, tfidf weights smaller %s=%s', prop,
        #    repr(m_tfidf).replace('\n', ''), len(indices_with_tokens),
        #    len(ser_tfidf), delta, len(ser_tfidf[ser_tfidf < delta]))

        return m_tfidf, list(indices_with_tokens)

    @staticmethod
    def _normalize_tfidf_matrix(
            m_tfidf: scipy.sparse.lil_matrix) -> scipy.sparse.csr_matrix:
        m_tmp = m_tfidf.power(2)
        m_tmp = m_tmp.sum(axis=1)
        # getA1() converts a matrix to a flattened ndarray
        m_tmp = np.sqrt(m_tmp.getA1())
        # np.where does not avoid RuntimeWarning, see:
        # https://stackoverflow.com/questions/64747346/numpy-divide-by-zero-encountered-in-true-divide-on-np-where
        m_tmp = np.where(m_tmp > 0, 1 / m_tmp, 0)

        m_diag = scipy.sparse.diags(m_tmp)
        m_tfidf_norm = m_diag * m_tfidf
        return m_tfidf_norm

    @staticmethod
    def dot_for_candidate_pairs(candidate_pairs: pd.MultiIndex,
                                 matrix: scipy.sparse.csr_matrix,
                                 size_1: int) -> np.ndarray:
        """Computes the dot product for sparse tfidf vectors of all candidate pairs.

        :param candidate_pairs: the set of candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param matrix: the sparse tfidf matrix, rows correspond to entities
        :type matrix: scipy.sparse.csr_matrix
        :param size_1: the size of the first dataset
        :type size_1: int
        :return: the dot product for all candidate pairs
        :rtype: np.ndarray
        """
        m_a = matrix[:size_1, ]
        l_idx1 = candidate_pairs.get_level_values(level=0)
        m_a_lifted = cast(scipy.sparse.csr_matrix, m_a[l_idx1])

        m_b = matrix[size_1:, ]
        l_idx2 = candidate_pairs.get_level_values(level=1) - size_1
        m_b_lifted = cast(scipy.sparse.csr_matrix, m_b[l_idx2])

        #logging.debug('m_a_lifted=%s, m_b_lifted=%s', m_a_lifted.shape, m_b_lifted.shape)
        # element-wise multiplication of two sparse matrices in scipy's csr format
        m_prod = m_a_lifted.multiply(m_b_lifted)
        #logging.debug('m_prod=%s', m_prod.shape)

        m_dot = m_prod.sum(axis=1)
        logging.debug('m_dot=%s, type=%s', m_dot.shape, type(m_dot))

         # getA1() converts a matrix to a flattened ndarray of shape (len(candidate_pairs),)
        return m_dot.getA1()


class SimilarityManager():
    """This class computes the similarity value for all candidate
    pairs and for all configured properties. It contains different
    stragegies for different similarity functions to compute the
    similarity values efficiently.
    """

    def __init__(self, df_data: pd.DataFrame, size_1: int,
                 candidate_pairs: pd.MultiIndex, property_mapping: List[dict],
                 embedding_batch_size: int, embedding_model: str,
                 embedding_device: str,
                 df_token_index: Optional[pd.DataFrame]) -> None:
        """Constructor

        :param df_data: the stacked data of the first and second dataset
        :type df_data: pd.DataFrame
        :param size_1: size of first dataset
        :type size_1: int
        :param candidate_pairs: set of all candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param property_mapping: defines which similarity function is used
            for which pair of properties
        :type property_mapping: List[dict]
        :param embedding_batch_size: batch size for embedding calculation
        :type embedding_batch_size: int
        :param embedding_model: language model of sentence transformer
        :type embedding_model: str
        :param embedding_device: device for embedding calculation, cpu or cuda
        :type embedding_device: str
        :param df_token_index: the token index is only used for similarity function 'tfidf'
        :type df_token_index: pd.DataFrame
        """
        self.df_data = df_data
        self.size_1 = size_1
        self.candidate_pairs = candidate_pairs
        self.property_mapping = property_mapping
        self.embedding_batch_size = embedding_batch_size
        self.embedding_model = embedding_model
        self.embedding_device = embedding_device

        self.df_token_index = None
        if df_token_index is not None:
            self.df_token_index = df_token_index.copy()
            self.df_token_index.index.names = ['token']
            self.df_token_index.reset_index(inplace=True)

        self.df_sim: pd.DataFrame

    def get_similarity_values(self) -> pd.DataFrame:
        """Returns the calculated similarity values as dataframe.
        Its index corresponds to the candidate pairs,
        its columns correspond to the similarity functions.

        :return: similarity values
        :rtype: pd.DataFrame
        """
        return self.df_sim

    def calculate_similarities(
            self,
            blocking_values: Optional[pd.Series] = None,
            blocking_vectors: Optional[np.ndarray] = None) -> pd.DataFrame:
        """Calculates the similarity values for all candidate pairs
        and for all configured pairs of properties and
        corresponding similarity functions.

        :param blocking_values: values for blocking properties, defaults to None
        :type blocking_values: Optional[pd.Series], optional
        :param blocking_vectors: vectors corresponding to values computed during blocking,
            defaults to None
        :type blocking_vectors: Optional[np.ndarray], optional
        :raises ValueError: if a similarity function is unknown
        :return: similarity values (see get_similarity_values())
        :rtype: pd.DataFrame
        """

        # split property mapping into categories that allow for different
        # efficient implementations for the calculation of the similarity values
        mapping_categories = collections.defaultdict(list)
        for mapping in self.property_mapping:
            name = mapping['sim_fct_name']
            if name == 'embedding':
                mapping_categories[name].append(mapping)
            elif name in ['tfidf', 'tfidf_sklearn']:
                mapping_categories['tfidf'].append(mapping)
            elif name in ['absolute', 'relative', 'equal']:
                mapping_categories['vectorized'].append(mapping)
            elif name in ['fuzzy']:
                mapping_categories['pseudo'].append(mapping)
            else:
                raise ValueError(f'unknown similarity function: {name}')

        # create empty dataframe for similarity values, sorted wrt. candidate pairs
        df_sim = pd.DataFrame(index=self.candidate_pairs)
        df_sim.index.set_names(['idx_1', 'idx_2'], inplace=True)
        matchain.util.sort_pairs(df_sim)

        property_mapping = mapping_categories['pseudo']
        if property_mapping:
            df_sim = SimilarityManager._compute_pseudo_vectorized_similarity(
                self.df_data, property_mapping, self.candidate_pairs, df_sim)

        property_mapping = mapping_categories['vectorized']
        if property_mapping:
            df_sim = SimilarityManager._compute_vectorized_similarity(
                self.df_data, property_mapping, self.candidate_pairs, df_sim)

        property_mapping = mapping_categories['tfidf']
        if property_mapping:
            size_2 = len(self.df_data) - self.size_1
            df_sim = SimilarityManager._compute_tfidf_similarity(
                self.size_1, size_2, property_mapping, self.candidate_pairs,
                df_sim, self.df_token_index, self.df_data)

        property_mapping = mapping_categories['embedding']
        if property_mapping:
            df_sim = SimilarityManager._compute_embedding_similarity(
                self.df_data, property_mapping, self.candidate_pairs, df_sim,
                self.embedding_model, self.embedding_device,
                self.embedding_batch_size, blocking_values, blocking_vectors)

        columns = sorted(df_sim.columns)
        self.df_sim = df_sim[columns]
        logging.debug('similarity values statistics:\n%s',
                      self.df_sim.describe())
        return self.df_sim

    @staticmethod
    def _compute_pseudo_vectorized_similarity(
            df_data: pd.DataFrame, property_mapping: List[dict],
            candidate_pairs: pd.MultiIndex,
            df_sim: pd.DataFrame) -> pd.DataFrame:
        """Computes the similarity values for all candidate pairs for each
        given property and adds the resulting columns to df_sim.
        The implementation is pseudo-vectorized; it uses a for-loop and
        is therefore not as efficient as a fully vectorized implementation.

        :param df_data: dataframe with the stacked data of the first and second dataset
        :type df_data: pd.DataFrame
        :param property_mapping: defines which similarity functions are used
        :type property_mapping: List[dict]
        :param candidate_pairs: set of all candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param df_sim: dataframe with already computed similarity values
        :type df_sim: pd.DataFrame
        :return: dataframe with additional new similarity values
        :rtype: pd.DataFrame
        """
        logging.info('computing pseudo-vectorized similarity')

        starttime = time.time()

        propmaps_pseudo = [
            m for m in property_mapping if m['sim_fct_name'] in ['fuzzy']
        ]
        if propmaps_pseudo:
            props = [t['prop1'] for t in propmaps_pseudo]
            np_array = df_data[props].to_numpy()
            lifted_array = AdvancedIndexHelper.lift_index_to_candidates(
                candidate_pairs, np_array)

            PseudoVectorizedSimilarity.compute_similarity(
                propmaps_pseudo, lifted_array, df_sim)

        prop_columns = [t['column_name'] for t in property_mapping]
        diff = time.time() - starttime
        logging.info(
            'computed pseudo-vectorized similarity=%s, sim columns=%s, time=%s',
            len(df_sim), prop_columns, diff)

        return df_sim

    @staticmethod
    def _compute_vectorized_similarity(df_data: pd.DataFrame,
                                       property_mapping: List[dict],
                                       candidate_pairs: pd.MultiIndex,
                                       df_sim: pd.DataFrame) -> pd.DataFrame:
        """Computes the similarity values for all candidate pairs for each given
        property and adds the resulting columns to df_sim.
        The similarity functions are expressed in terms of numpy which allows
        to implement this method in a vectorized and therefore efficient way.

        :param df_data: dataframe with the stacked data of the first and second dataset
        :type df_data: pd.DataFrame
        :param property_mapping: defines which similarity functions are used
        :type property_mapping: List[dict]
        :param candidate_pairs: set of all candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param df_sim: dataframe with already computed similarity values
        :type df_sim: pd.DataFrame
        :return: dataframe with additional new similarity values
        :rtype: pd.DataFrame
        """
        logging.info('computing vectorized similarity')

        starttime = time.time()

        props = [t['prop1'] for t in property_mapping]
        np_array = df_data[props].to_numpy()
        lifted_array = AdvancedIndexHelper.lift_index_to_candidates(
            candidate_pairs, np_array)

        for i_prop, propmap in enumerate(property_mapping):
            sim_fct_name = propmap['sim_fct_name']
            params_sim_fct = propmap['sim_fct_params']
            cut_off_value = params_sim_fct.get('cut_off_value', 1)
            sim_fct = VectorizedSimilarity.create_similarity_function_from_params(
                sim_fct_name, cut_off_value)
            v = lifted_array[:, 2 * i_prop]
            w = lifted_array[:, 2 * i_prop + 1]
            # v and w have the same size as the vector of candidate pairs and
            # the order of their elements corresponds to the order of the candidate pairs.
            # This allows a fast vectorized application of the similarity function.
            result = sim_fct(v, w)
            col = propmap['column_name']
            df_sim[col] = result

        prop_columns = [t['column_name'] for t in property_mapping]
        diff = time.time() - starttime
        logging.info(
            'computed vectorized similarity=%s, sim columns=%s, time=%s',
            len(df_sim), prop_columns, diff)

        return df_sim

    @staticmethod
    def _compute_tfidf_similarity(
            size_1: int, size_2: int, property_mapping: List[dict],
            candidate_pairs: pd.MultiIndex, df_sim: pd.DataFrame,
            df_token_index: pd.DataFrame, df_data: pd.DataFrame) -> pd.DataFrame:
        """Computes the similarity values for all candidate pairs
        for each given property and adds the resulting columns to df_sim.
        The implementation is specific to the tfidf similarity function and
        uses sparse matrix operations.

        :param size_1: size of the first dataset
        :type size_1: int
        :param size_2: size of the second dataset
        :type size_2: int
        :param property_mapping: defines which similarity functions are used
        :type property_mapping: List[dict]
        :param candidate_pairs: set of all candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param df_sim: dataframe with already computed similarity values
        :type df_sim: pd.DataFrame
        :param df_token_index: dataframe with the token index
        :type df_token_index: pd.DataFrame
        :param df_data: dataframe with the stacked data of the first and second dataset
        :type df_data: pd.DataFrame
        :return: dataframe with additional new similarity values
        :rtype: pd.DataFrame
        """
        logging.info('computing similarity tfidf')
        starttime = time.time()

        prop_columns = []
        for propmap in property_mapping:

            prop = propmap['prop1']
            column = propmap['column_name']
            prop_columns.append(column)
            name = propmap['sim_fct_name']
            if name == 'tfidf_sklearn':
                # use shingles of strings to create sparse vectors with tfidf weights
                str_series = df_data[prop]
                str_series = str_series.fillna('', inplace=False)
                wrapper = ShingleWrapper(shingle_size=3)
                tfidf_matrix = wrapper.generate_vectors(str_series)
                df_sim[column] = TfidfSimilarity.dot_for_candidate_pairs(
                    candidate_pairs, tfidf_matrix, size_1)

            elif name == 'tfidf':
                # use tokens (words) of string to create sparse vectors with tfidf weights
                # this is faster implementation of the code originally used for the autocal
                # the implemention ensures to obtain the same results
                max_idf = propmap['sim_fct_params']['maxidf']
                TfidfSimilarity.compute_similarity(prop, column, df_token_index,
                                                candidate_pairs, df_sim, size_1,
                                                size_2, max_idf)

        diff = time.time() - starttime
        logging.info('computed similarity tfidf=%s, sim columns=%s, time=%s',
                     len(df_sim), prop_columns, diff)
        return df_sim

    @staticmethod
    def _compute_embedding_similarity(
            df_data: pd.DataFrame,
            property_mapping: List[dict],
            candidate_pairs: pd.MultiIndex,
            df_sim: pd.DataFrame,
            model: str,
            device: str,
            batch_size: int,
            blocking_values: Optional[pd.Series] = None,
            blocking_vectors: Optional[np.ndarray] = None) -> pd.DataFrame:
        """Computes the similarity values for all candidate pairs
        for each given property and adds the resulting columns to df_sim.
        The implementation is specific to the embedding similarity function and
        reformulates the computation as a matrix multiplication in numpy.

        :param df_data: dataframe with the stacked data of the first and second dataset
        :type df_data: pd.DataFrame
        :param property_mapping: defines which similarity functions are used
        :type property_mapping: List[dict]
        :param candidate_pairs: set of all candidate pairs
        :type candidate_pairs: pd.MultiIndex
        :param df_sim: dataframe with already computed similarity values
        :type df_sim: pd.DataFrame
        :param model: language model used for computing the embeddings
        :type model: str
        :param device: device used for computing the embeddings
        :type device: str
        :param batch_size: batch size used for computing the embeddings
        :type batch_size: int
        :param blocking_values: values created for blocking properties
        :type blocking_values: Optional[pd.Series], optional
        :param blocking_vectors: vectors for values created for blocking properties
        :type blocking_vectors: Optional[np.ndarray], optional
        :return: dataframe with additional new similarity values
        :rtype: pd.DataFrame
        """
        logging.info('preparing embeddings')
        starttime = time.time()
        props = [t['prop1'] for t in property_mapping]
        prop_columns = [t['column_name'] for t in property_mapping]

        new_values, index_array = AdvancedIndexHelper.create_index_array_from_property_values(
            df_data, props, blocking_values)
        logging.info('prepared embeddings, time=%s', time.time() - starttime)

        if len(new_values) == 0 and blocking_vectors is not None:
            logging.info('embeddings were already computed during blocking')
            all_vectors = blocking_vectors
        else:
            logging.info('creating embeddings')
            starttime = time.time()
            wrapper = SentenceTransformerWrapper(model, device, batch_size)
            new_vectors = wrapper.generate_vectors(new_values)
            logging.info('created embeddings, time=%s',
                         time.time() - starttime)

            if blocking_vectors is None:
                all_vectors = new_vectors
            else:
                all_vectors = np.concatenate((blocking_vectors, new_vectors),
                                             axis=0)

        logging.debug('computing embedding similarity')
        starttime = time.time()
        df_cosine_sim = AdvancedIndexHelper.compute_cosine_similarity(
            candidate_pairs, index_array, prop_columns, all_vectors)
        df_sim[prop_columns] = df_cosine_sim
        logging.info(
            'computed embedding similarity=%s, sim columns=%s, time=%s',
            len(df_sim), prop_columns,
            time.time() - starttime)
        return df_sim


def run(config: dict, df_data: pd.DataFrame, size_1: int,
        candidate_pairs: pd.MultiIndex,
        token_index_unpruned: Optional[pd.DataFrame],
        blocking_values: Optional[pd.Series],
        blocking_vectors: Optional[np.ndarray]
        ) -> Tuple[pd.DataFrame, List[dict]]:
    """Entry point for the similarity computation as part of the command chain."""

    params_mapping = config['dataset']['props_sim']
    maxidf = config['similarity']['tfidf_maxidf']
    property_mapping = create_property_mapping(params_mapping, maxidf)

    embedding_batch_size = config['similarity']['embedding_batch_size']
    embedding_model = config['similarity']['embedding_model']
    embedding_device = config['similarity']['embedding_device']

    # unpruned token index is used here to get the same tfidf cosine similarity scores
    # as in the OntoMatch code
    sim_manager = SimilarityManager(df_data,
                                    size_1,
                                    candidate_pairs,
                                    property_mapping,
                                    embedding_batch_size=embedding_batch_size,
                                    embedding_model=embedding_model,
                                    embedding_device=embedding_device,
                                    df_token_index=token_index_unpruned)
    df_sim = sim_manager.calculate_similarities(blocking_values,
                                                blocking_vectors)
    return df_sim, property_mapping
