# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/3.4_facade.ipynb (unless otherwise specified).

__all__ = ['TraceLinkValue', 'NumDoc', 'VocabSize', 'AverageToken', 'Vocab', 'VocabShared', 'SharedVocabSize',
           'MutualInformation', 'CrossEntropy', 'KLDivergence', 'get_docs', 'get_counters', 'preprocess']

# Cell
def TraceLinkValue(source, target, technique):
    if (technique == "VSM"):
        pass
    if (technique == "LDA"):
        pass
    if (technique == "orthogonal"):
        pass
    if (technique == "LSA"):
        pass
    if (technique == "JS"):
        pass
    if (technique == "word2vec"):
        pass
    if (technique == "doc2vec"):
        pass
    value = random.randint(0,1)/100

    return value

#export
def NumDoc(source, target):
    source_doc = source.shape[0]
    target_doc = target.shape[0]
    difference = source_doc - target_doc
    return [source_doc, target_doc, difference, -difference]

#export
def VocabSize(source, target):
    source_list = preprocess(source)
    target_list = preprocess(target)
    source_size = len(source_list[0])
    target_size = len(target_list[0])
    difference = source_size - target_size
    return [source_size, target_size, difference, -difference]

#export
def AverageToken(source, target):
    source_doc = source.shape[0]
    target_doc = target.shape[0]

    source_list = preprocess(source)
    target_list = preprocess(target)

    source_total_token = sum(source_list[0].values())
    target_total_token = sum(target_list[0].values())

    source_token = source_total_token/source_doc
    target_token = target_total_token/target_doc
    difference = source_token - target_token
    return [source_token, target_token, difference, -difference]

#export
def Vocab(artifacts_df):
    #we can add a parameter for user to specify the number of most frequent token to return
    cnts = preprocess(artifacts_df)
    vocab_list = cnts[0].most_common(3)
    total = sum(cnts[0].values())
    vocab_dict = dict()
    vocab_dict[vocab_list[0][0]] = [vocab_list[0][1], vocab_list[0][1]/total]
    vocab_dict[vocab_list[1][0]] = [vocab_list[1][1], vocab_list[1][1]/total]
    vocab_dict[vocab_list[2][0]] = [vocab_list[2][1], vocab_list[2][1]/total]

    return vocab_dict


#export
def VocabShared(source, target):
    df = pd.concat([source, target])
    return Vocab(df)

#export
def SharedVocabSize(source, target):
    df = pd.concat([source, target])
    df_counts = preprocess(df)
    shared_size = len(df_counts[0])
    return shared_size

#export
def MutualInformation(source, target):
    mutual_information = random.randint(100,200)
    return mutual_information

#export
def CrossEntropy(source, target):
    cross_entropy = random.randint(100,200)
    cross_entropy = get_system_entropy_from_df(source, "col1",)
    return cross_entropy

#export
def KLDivergence(source, target):
    divergence = random.randint(100,200)
    return divergence

#export
def get_docs(df, spm):
    docs = []
    for fn in df["col1"]:
        docs += spm.EncodeAsPieces(fn)
    return docs

#export
def get_counters(docs):
    doc_cnts = []
    cnt = Counter()
    for tok in docs:
        cnt[tok] += 1
        doc_cnts.append(cnt)
    return doc_cnts

#export
def preprocess(artifacts_df):
    spm = sp.SentencePieceProcessor()
    output = Path('test_data\models')
    system_name = "test"
    spm.Load(f"{output / system_name}.model")
    docs = get_docs(artifacts_df,spm)
    cnts = get_counters(docs)
    return cnts