{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import codecs\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cltk.tokenize.word import WordTokenizer\n",
    "old_norse_word_tokenizer = WordTokenizer(\"old_norse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(os.getenv(\"USER\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\clement_besnier'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.environ['USERPROFILE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "expected str, bytes or os.PathLike object, not NoneType",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-7d08bd6cd0cc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m CLTK_DATA_DIRECTORY = os.path.join(os.getenv(\"HOME\"), \"cltk_data\", \"old_norse\", \"text\", \"old_norse_text_perseus\", \"plain_text\", \n\u001b[1;32m----> 2\u001b[1;33m                                    \"Völsunga_saga\")\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Programs\\Python\\Python36-32\\lib\\ntpath.py\u001b[0m in \u001b[0;36mjoin\u001b[1;34m(path, *paths)\u001b[0m\n\u001b[0;32m     73\u001b[0m \u001b[1;31m# Join two (or more) paths.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mpaths\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 75\u001b[1;33m     \u001b[0mpath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfspath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     76\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbytes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m         \u001b[0msep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34mb'\\\\'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: expected str, bytes or os.PathLike object, not NoneType"
     ]
    }
   ],
   "source": [
    "CLTK_DATA_DIRECTORY = os.path.join(os.getenv(\"HOME\"), \"cltk_data\", \"old_norse\", \"text\", \"old_norse_text_perseus\", \"plain_text\", \n",
    "                                   \"Völsunga_saga\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "CLTK_DATA_DIRECTORY = os.path.join(os.environ['USERPROFILE'], \"cltk_data\", \"old_norse\", \"text\", \"old_norse_text_perseus\", \"plain_text\", \n",
    "                                   \"Völsunga_saga\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "chapters = []\n",
    "l_filenames = os.listdir(CLTK_DATA_DIRECTORY)\n",
    "l_filenames = sorted(l_filenames, key=lambda x: x.split(\"_\")[1].split(\".\")[0])\n",
    "for filename in l_filenames:\n",
    "    with codecs.open(os.path.join(CLTK_DATA_DIRECTORY, filename), \"r\", encoding=\"utf-8\") as f:\n",
    "        chapters.append(f.read().split(\". \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_chapters = []\n",
    "for i, chapter in enumerate(chapters):\n",
    "    tokenized_chapters.append([])\n",
    "    for sentence in chapter:\n",
    "        tokenized_chapters[i].append(old_norse_word_tokenizer.tokenize(sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['HÉR', 'hefr', 'upp', 'ok', 'segir', 'frá', 'þeim', 'manni', ',', 'er', 'Sigi', 'ernefndr', 'ok', 'kallaðr', ',', 'at', 'héti', 'sonr', 'Óðins']\n"
     ]
    }
   ],
   "source": [
    "print(tokenized_chapters[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vecinput = [line for chapter in tokenized_chapters for line in chapter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\clement_besnier\\pycharmprojects\\comparison_sigurdr_siegfried\\venv\\lib\\site-packages\\gensim\\models\\base_any2vec.py:743: UserWarning: C extension not loaded, training will be slow. Install a C compiler and reinstall gensim for fast training.\n",
      "  \"C extension not loaded, training will be slow. \"\n"
     ]
    }
   ],
   "source": [
    "model = Word2Vec(word2vecinput, min_count=1, size=50)\n",
    "model.save(\"old_norse_word2vec_model\")\n",
    "model = Word2Vec.load(\"old_norse_word2vec_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.models.word2vec.Word2VecVocab at 0xb1e6f70>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.4338045 , -0.73307973,  2.1021817 ,  1.1929849 , -1.1417882 ,\n",
       "        0.68206334,  0.07491098, -0.25942743,  2.5735452 , -0.5124804 ,\n",
       "       -0.49905834,  0.14490788, -0.2243631 , -0.26734868,  1.2562293 ,\n",
       "       -1.1485595 ,  0.72450525,  0.49183893,  0.7781776 ,  0.0303275 ,\n",
       "       -0.75544894, -0.12640889, -0.35750374,  0.3556944 ,  1.25945   ,\n",
       "        1.275915  , -1.0897819 , -0.8399024 ,  0.50003505,  0.35015535,\n",
       "        0.19254424, -1.1115782 , -1.3457205 , -1.2196482 ,  1.1242206 ,\n",
       "        1.657532  , -0.55481523, -0.01611723, -0.97643673,  0.5304799 ,\n",
       "        0.18681969, -0.38283703,  0.6232244 , -0.81129575, -0.1486712 ,\n",
       "        0.20537816,  0.51889247,  2.1644828 , -0.5209852 , -1.114733  ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv['ok']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9870095"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.similarity(\"konu\", \"manni\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7175553"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv.similarity(\"konungr\", \"hefr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
