#AUTOGENERATED! DO NOT EDIT! File to edit: dev/predict.ipynb (unless otherwise specified).

__all__ = ['get_embedding', 'load_model']

#Cell
# !pip install keras
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import numpy as np
# !pip install gensim
from gensim.models import KeyedVectors
def get_embedding(text, maxlen = 100, max_words = 10000, w2v_path = 'output/zh.vec'):

    # text to seq

    tokenizer = Tokenizer(num_words=max_words)
    tokenizer.fit_on_texts(text)
    sequences = tokenizer.texts_to_sequences(df.text)

    print('The obejct is transfromed to %s' % str(type(sequences)))
    print('The original length of seqences is %s' % str([len(sequence) for sequence in sequences[:5]]))

    data = pad_sequences(sequences, maxlen=maxlen, value = 0.0)

    print('The transformed length of seqences is %s after padding' % str([len(sequence) for sequence in data[:5]]))
    # 长句子被剪裁(`maxlen`)，短句子被 0 padding

    # seq output

    print('The shape output is %s respect to the maxlen arg %s' % (data.shape, maxlen))

    word_index = tokenizer.word_index

    print('The word dict class %s' % type(word_index))

    print('Some like %s' % [str(key)+": "+str(value) for key, value in word_index.items()][:5])


    # embedding

    zh_model = KeyedVectors.load_word2vec_format(w2v_path)

    print('The pretrained word2vec size is %s' % str(zh_model.vectors.shape))
    print('with some words like %s' % list(iter(zh_model.vocab))[:5])

    embedding_dim = len(zh_model[next(iter(zh_model.vocab))])
    print('Thus the embedding size is %s' % embedding_dim)
    print('Within it, ')
    print('max: ',zh_model.vectors.max())
    print('min: ',zh_model.vectors.min())

    embedding_matrix = np.random.uniform(zh_model.vectors.min(), zh_model.vectors.max(), [max_words, embedding_dim])

    # 随机数参考 https://stackoverflow.com/questions/11873741/sampling-random-floats-on-a-range-in-numpy

    embedding_matrix = (embedding_matrix - 0.5) * 2

    # zh_model.get_vector("的").shape
    # zh_model.get_vector("李").shape

    for word, i in word_index.items():
        if i < max_words:
            try:
                embedding_vector = zh_model.get_vector(word)
                embedding_matrix[i] = embedding_vector
            except:
                pass # 如果无法获得对应的词向量，我们就干脆跳过，使用默认的随机向量。

    # 这也是为什么，我们前面尽量把二者的分布调整成一致。

    print('The output embedding size is %s' % str(embedding_matrix.shape))

    return (data, embedding_matrix)

#Cell
import keras
def load_model(path):
     return keras.models.load_model(path)