{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!pip install gensim\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading a Model\n",
    "\n",
    "Google trained the Word2Vec skipgram model on a ton of Google News articles  \n",
    "Model is 3G uncompressed so this will take a couple minutes, go get a coffee...  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wv = Word2Vec.load_word2vec_format('/home/hobson/Downloads/GoogleNews-vectors-negative300.bin.gz', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'queen', 0.711819589138031),\n",
       " (u'monarch', 0.6189677119255066),\n",
       " (u'princess', 0.5902431011199951),\n",
       " (u'crown_prince', 0.5499463081359863)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['king', 'woman'], negative=['man'])[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'princess', 0.7041179537773132),\n",
       " (u'duchess', 0.6229751110076904),\n",
       " (u'monarch', 0.6019776463508606),\n",
       " (u'Queen_Consort', 0.5901831984519958)]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['prince', 'woman'], negative=['man'])[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'Royalty', 0.5377441644668579),\n",
       " (u'lords', 0.5285096168518066),\n",
       " (u'royalties', 0.49976375699043274),\n",
       " (u'queen', 0.49964314699172974),\n",
       " (u'princess', 0.49909472465515137),\n",
       " (u'Queen_Consort', 0.4874693751335144),\n",
       " (u'vampires_lurking', 0.4812909960746765),\n",
       " (u'handmaid', 0.48073622584342957),\n",
       " (u'noblemen', 0.4779472053050995),\n",
       " (u'very_pampered_McElhatton', 0.47529909014701843)]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['lord', 'royalty', 'woman'], negative=['man'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'manor_house', 0.566051185131073),\n",
       " (u'bedchamber', 0.5568591952323914),\n",
       " (u'lords', 0.555293083190918),\n",
       " (u'castle', 0.5414100885391235),\n",
       " (u'Viscountess', 0.5305901169776917),\n",
       " (u'baroness', 0.52792888879776),\n",
       " (u'aristo', 0.5237292647361755),\n",
       " (u'chatelaine', 0.5204670429229736),\n",
       " (u'gentlewoman', 0.5187667608261108),\n",
       " (u'manors', 0.5181090831756592)]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['lord', 'manor', 'woman'], negative=['man'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'king', 0.6958589553833008),\n",
       " (u'kings', 0.5950948596000671),\n",
       " (u'queens', 0.5838503241539001),\n",
       " (u'monarch', 0.5398427844047546)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar(positive=['queen', 'man'], negative=['woman'])[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quantified Self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ">>> list(w for w in wv.vocab if w.lower().startswith('quantified_'))\n",
    "[u'quantified_emission',\n",
    " u'Quantified_Publisher',\n",
    " u'Quantified_Marketing_Group',\n",
    " u'Quantified_Analysis',\n",
    " u'Quantified_Publisher_program',\n",
    " u'Quantified_Self']\n",
    ">>> wv.most_similar(positive=['Quantified_Self', 'journal', 'diary'])\n",
    "[(u'diaries', 0.679690957069397),\n",
    " (u'journals', 0.6234577894210815),\n",
    " (u'journal_Genome_Biology', 0.5596049427986145),\n",
    " (u'Preventing_Chronic_Disease', 0.5553246140480042),\n",
    " (u'journal_Rheumatology', 0.5511202216148376),\n",
    " (u'diary_entries', 0.5450901389122009),\n",
    " (u'Personall_warns', 0.5413833856582642),\n",
    " (u'journal_PNAS', 0.5409928560256958),\n",
    " (u'journal_Menopause', 0.5374890565872192),\n",
    " (u'journal_PLOS', 0.5371968150138855)]\n",
    ">>> wv.most_similar(positive=['journal', 'diary', 'python', 'semantic', 'NLP'])\n",
    "[(u'journal_PNAS', 0.5335176587104797),\n",
    " (u'journals', 0.5311996340751648),\n",
    " (u'Personall_warns', 0.5287125110626221),\n",
    " (u'diaries', 0.5208494663238525),\n",
    " (u'Connotea', 0.5163055062294006),\n",
    " (u'PloS_Biology', 0.5094947218894958),\n",
    " (u'Wikipedia_encyclopedia', 0.5085359811782837),\n",
    " (u'journal_Neuropsychology', 0.5040772557258606),\n",
    " (u'journal_Behavioural_Ecology', 0.5029593110084534),\n",
    " (u'ontology', 0.5026495456695557)]\n",
    ">>> wv.most_similar(positive=['python', 'semantic_processing', 'meaning', 'NLP'])\n",
    ">>> wv.most_similar(positive=['python', 'semantic', 'processing', 'meaning', 'NLP'])\n",
    "[(u'inferencing', 0.6100273132324219),\n",
    " (u'regex', 0.5578750371932983),\n",
    " (u'semantics', 0.5554606914520264),\n",
    " (u'semantic_indexing', 0.5523117184638977),\n",
    " (u'PCRE', 0.5505284667015076),\n",
    " (u'semantically', 0.543846845626831),\n",
    " (u'XML_namespaces', 0.5346047282218933),\n",
    " (u'Extensible_Application_Markup_Language', 0.5326448082923889),\n",
    " (u'phonemic', 0.5311585068702698),\n",
    " (u'Boolean_operators', 0.5305431485176086)]\n",
    ">>> wv.most_similar(positive=['Python', 'python', 'semantic', 'processing', 'meaning', 'NLP'])\n",
    "[(u'inferencing', 0.6243683099746704),\n",
    " (u'PCRE', 0.6188617944717407),\n",
    " (u'regex', 0.6080276370048523),\n",
    " (u'parser', 0.5828561186790466),\n",
    " (u'XML_namespaces', 0.5820693373680115),\n",
    " (u'XML_parser', 0.5805317759513855),\n",
    " (u'XML_syntax', 0.5791804790496826),\n",
    " (u'TSQL', 0.5776033997535706),\n",
    " (u'DataWindow', 0.5756461024284363),\n",
    " (u'NET_CLR', 0.5726565718650818)]\n",
    ">>> wv.most_similar(positive=['Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install'])\n",
    "[(u'NET_CLR', 0.6145628094673157),\n",
    " (u'PCRE', 0.6098397374153137),\n",
    " (u'swf_file', 0.6015275120735168),\n",
    " (u'TITLE_Debian_update', 0.5981236696243286),\n",
    " (u'inferencing', 0.5910664200782776),\n",
    " (u'XML_parser', 0.5868363380432129),\n",
    " (u'PHP_scripting', 0.5843981504440308),\n",
    " (u'HTML_CSS_JavaScript', 0.583228588104248),\n",
    " (u'remoting', 0.5813583731651306),\n",
    " (u'regex', 0.5790796279907227)]\n",
    ">>> wv.most_similar(positive=['Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular_expression'])\n",
    ">>> wv.most_similar(positive=['Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression'])\n",
    "[(u'TSQL', 0.46374911069869995),\n",
    " (u'XML_XSLT', 0.46319350600242615),\n",
    " (u'HTML_CSS_JavaScript', 0.46195775270462036),\n",
    " (u'NET_CLR', 0.4589165151119232),\n",
    " (u'VisualBasic', 0.4558500051498413),\n",
    " (u'NET_framework', 0.454326868057251),\n",
    " (u'Perl_Python_Ruby', 0.4509257972240448),\n",
    " (u'inferencing', 0.44831690192222595),\n",
    " (u'mod_rewrite', 0.4465762674808502),\n",
    " (u'swf_file', 0.4453127682209015)]\n",
    ">>> wv.most_similar(positive=['Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft])\n",
    ">>> wv.most_similar(positive=['open_source', 'Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft'])\n",
    ">>> wv.most_similar(positive=['Open_Source', 'Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft'])\n",
    "[(u'XML_XSLT', 0.4880766272544861),\n",
    " (u'HTML_CSS_JavaScript', 0.4751140773296356),\n",
    " (u'Java_JSP', 0.46583420038223267),\n",
    " (u'inferencing', 0.4654781222343445),\n",
    " (u'Perl_Python', 0.4650688171386719),\n",
    " (u'Perl_Python_Ruby', 0.4642208516597748),\n",
    " (u'TSQL', 0.458888977766037),\n",
    " (u'Actionscript', 0.45879411697387695),\n",
    " (u'mod_rewrite', 0.4582417905330658),\n",
    " (u'NET_framework', 0.4580802023410797)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'OpenCog', 'Open_Source', 'Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft'])\n",
    "[(u'XML_XSLT', 0.5321134924888611),\n",
    " (u'Java_JSP', 0.5281328558921814),\n",
    " (u'inferencing', 0.5163978338241577),\n",
    " (u'HTML_CSS_JavaScript', 0.512594997882843),\n",
    " (u'Actionscript', 0.5115547180175781),\n",
    " (u'Object_Oriented', 0.5094218850135803),\n",
    " (u'Query_Language', 0.508894145488739),\n",
    " (u'mod_rewrite', 0.5063719749450684),\n",
    " (u'MochiKit', 0.5052181482315063),\n",
    " (u'NET_CLR', 0.5049869418144226)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL'])\n",
    "[(u'inferencing', 0.4273879826068878),\n",
    " (u'Assembler', 0.4221707284450531),\n",
    " (u'Literal_translation', 0.42108485102653503),\n",
    " (u'XML_XSLT', 0.420962393283844),\n",
    " (u'Java_JSP', 0.4199409484863281),\n",
    " (u'HTML_CSS_JavaScript', 0.4166453182697296),\n",
    " (u'Schrodinger', 0.416521817445755),\n",
    " (u'Actionscript', 0.4158591628074646),\n",
    " (u'Semantic', 0.4158053696155548),\n",
    " (u'Perl_Python', 0.41291189193725586)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'Natural_Language_Processing', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'natural_language', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chatterbot', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL'])\n",
    "[(u'Bing_Translator', 0.5751106142997742),\n",
    " (u'Python_programming', 0.5679047703742981),\n",
    " (u'inferencing', 0.5638830065727234),\n",
    " (u'processing_NLP', 0.5546227693557739),\n",
    " (u'XML_XSLT', 0.5532411336898804),\n",
    " (u'chatbots', 0.5529900193214417),\n",
    " (u'WordNet', 0.5514096617698669),\n",
    " (u'HTML_CSS_JavaScript', 0.5490512847900391),\n",
    " (u'HTML_CSS_Javascript', 0.5486903190612793),\n",
    " (u'mathematical_notation', 0.5454741716384888)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL', 'Bing'])\n",
    "[(u'inferencing', 0.5493630170822144),\n",
    " (u'processing_NLP', 0.5458688735961914),\n",
    " (u'XML_XSLT', 0.5446382761001587),\n",
    " (u'Python_programming', 0.542436957359314),\n",
    " (u'phonemic', 0.5351700186729431),\n",
    " (u'Bing_Translator', 0.533650279045105),\n",
    " (u'Ontologies', 0.5321763157844543),\n",
    " (u'Java_JSP', 0.5291105508804321),\n",
    " (u'Object_Oriented', 0.5287489295005798),\n",
    " (u'mathematical_notation', 0.5272959470748901)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['regex', 'regular', 'expression', 'Microsoft', 'TSQL', 'Bing', 'translate'])\n",
    "[(u'inferencing', 0.5338713526725769),\n",
    " (u'XML_XSLT', 0.5273652076721191),\n",
    " (u'Ontologies', 0.5210939049720764),\n",
    " (u'processing_NLP', 0.5178764462471008),\n",
    " (u'Python_programming', 0.5158846378326416),\n",
    " (u'chatbots', 0.5153536200523376),\n",
    " (u'Java_JSP', 0.5145242214202881),\n",
    " (u'Bing_Translator', 0.5135848522186279),\n",
    " (u'Object_Oriented', 0.5110706090927124),\n",
    " (u'mathematical_notation', 0.5103068351745605)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'pip', 'install', 'grammar'], negative=['statistical'])\n",
    "[(u'Python_programming', 0.6736586093902588),\n",
    " (u'PHP_scripting', 0.6660847663879395),\n",
    " (u'Bing_Translator', 0.6640334725379944),\n",
    " (u'regex', 0.6583566069602966),\n",
    " (u'NET_CLR', 0.6578915119171143),\n",
    " (u'semantic_markup', 0.657026469707489),\n",
    " (u'HTML_CSS_Javascript', 0.6541556715965271),\n",
    " (u'XML_syntax', 0.6483075618743896),\n",
    " (u'TITLE_Debian_update', 0.6467106938362122),\n",
    " (u'XML_XSLT', 0.6464080810546875)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'pip', 'install', 'statistical'], negative=['grammar'])\n",
    "[(u'NET_CLR', 0.6777916550636292),\n",
    " (u'Bing_Translator', 0.6773670315742493),\n",
    " (u'PHP_scripting', 0.6669802665710449),\n",
    " (u'TITLE_Debian_update', 0.6665417551994324),\n",
    " (u'inferencing', 0.6656586527824402),\n",
    " (u'HTML_CSS_Javascript', 0.6646273136138916),\n",
    " (u'regex', 0.6629053354263306),\n",
    " (u'Python_programming', 0.6611731648445129),\n",
    " (u'NET_Remoting', 0.6532155871391296),\n",
    " (u'ASPX', 0.6508787870407104)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'statistical'], negative=['grammar'])\n",
    "[(u'Bing_Translator', 0.6794119477272034),\n",
    " (u'inferencing', 0.6737903356552124),\n",
    " (u'regex', 0.669569194316864),\n",
    " (u'Python_programming', 0.6642642617225647),\n",
    " (u'HTML_CSS_Javascript', 0.6585268378257751),\n",
    " (u'XML_syntax', 0.6508367657661438),\n",
    " (u'semantic_indexing', 0.6507266759872437),\n",
    " (u'NET_CLR', 0.6478333473205566),\n",
    " (u'XML_eXtensible_Markup_Language', 0.6444374322891235),\n",
    " (u'Declarative', 0.6437892317771912)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'NLTK', 'statistical'], negative=['grammar'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'NLP', 'nltk', 'statistical'], negative=['grammar'])\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'statistical'], negative=['grammar'])\n",
    "[(u'Bing_Translator', 0.6709335446357727),\n",
    " (u'regex', 0.6649294495582581),\n",
    " (u'inferencing', 0.6554071307182312),\n",
    " (u'Python_programming', 0.6519725322723389),\n",
    " (u'NET_CLR', 0.6503692269325256),\n",
    " (u'HTML_CSS_Javascript', 0.6472666263580322),\n",
    " (u'ASPX', 0.6460676789283752),\n",
    " (u'semantic_indexing', 0.6457715630531311),\n",
    " (u'PHP_scripting', 0.6413751244544983),\n",
    " (u'XML_syntax', 0.6399379968643188)]\n",
    ">>> wv.most_similar(positive=['Cyc', 'Cog', 'Open_Source', 'Python', 'python', 'semantic', 'Semantic', 'language', 'ai', 'chatbot', 'chat', 'processing', 'meaning', 'statistical', 'semantic_indexing'], negative=['grammar'])\n",
    "[(u'Bing_Translator', 0.686427116394043),\n",
    " (u'regex', 0.6840069890022278),\n",
    " (u'inferencing', 0.6776837110519409),\n",
    " (u'NET_CLR', 0.6666378378868103),\n",
    " (u'ASPX', 0.6647640466690063),\n",
    " (u'TITLE_Debian_update', 0.6628689765930176),\n",
    " (u'PHP_scripting', 0.6614559888839722),\n",
    " (u'Python_programming', 0.6613916158676147),\n",
    " (u'HTML_CSS_Javascript', 0.6609216332435608),\n",
    " (u'XML_eXtensible_Markup_Language', 0.6534264087677002)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Journal Summarization\n",
    "\n",
    "Let's use state-of-the art Latent Dirichlet Allocation to preview a bunch of text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pug.nlp import util\n",
    "from gensim.summarization import summarize\n",
    "import os.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'accessed': datetime.datetime(2012, 6, 15, 22, 43, 6),\n",
       " 'created': datetime.datetime(2016, 4, 7, 17, 42, 56, 22474),\n",
       " 'dir': '/home/hobson/Downloads/evernote_notes',\n",
       " 'mode': 33252,\n",
       " 'modified': datetime.datetime(2012, 6, 15, 22, 43, 6),\n",
       " 'name': \"android g1 unlock and activate using friends tmobile sim card and your own provider's data plan.txt\",\n",
       " 'path': \"/home/hobson/Downloads/evernote_notes/android g1 unlock and activate using friends tmobile sim card and your own provider's data plan.txt\",\n",
       " 'size': 2936,\n",
       " 'type': 'file'}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree = util.find_files('/home/hobson/Downloads/evernote_notes/', ext='.txt')\n",
    "print(len(tree))\n",
    "tree[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `summarize`\n",
    "\n",
    "<img src=\"summarization_tutorial_plot.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 unlock and activate using friends tmobile sim card and your own provider's data plan\n",
      "just they QDo\n",
      "If you purchased a new G1 and you don’t want to be stuck with T-Mobile for 2 years, there are a couple of possible ways to bypass the initial activation so you can use the G1 on your AT&T or other carrier’s SIM card.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### remote, sms access to gps tracker in android or instamapper or gpstracker\n",
      "MISMATCHED VOCAB\n",
      "http://www.mobyko.com/\n",
      "http://www.smstxtbox.com/web/\n",
      "http://www.junglesms.com/\n",
      "http://www.free--sms.com/index.go\n",
      "http://www.pimpmysms.com/\n",
      "http://www.phones.com/\n",
      "http://www.devinosms.com/\n",
      "http://www.themobiweb.com/en/sms.html\n",
      "http://www.mobik.com/mobik/client/\n",
      "http://www.smsdiscount.com/en/index.html\n",
      "http://www.sendsmsnow.com/\n",
      "http://www.atrochatro.com/\n",
      "http://www.gsmvault.com/\n",
      "http://www.d1g.com/\n",
      "http://www.vazu.com/\n",
      "http://www.agentsms.com/\n",
      "http://www.yellowpages.com.eg/\n",
      "http://www.resalh.com/\n",
      "http://www.sms.com/\n",
      "http://www.shortmessage.com/\n",
      "http://www.uaesms.com/\n",
      "http://www.mobizone.com/\n",
      "http://www.mobizone.com/\n",
      "http://www.smspop.com/\n",
      "http://www.nemra1.com/\n",
      "http://www.boswtol.com/\n",
      "http://www.boswtol.com/\n",
      "http://www.itsalat.com/\n",
      "http://www.itsalat.com/\n",
      "http://www.quios.com/\n",
      "http://www.quios.com/\n",
      "http://www.clickatell.com/\n",
      "http://www.clickatell.com/\n",
      "http://www.freesms.com/\n",
      "http://www.freesms.com/\n",
      "http://www.free-sms.com/\n",
      "http://www.free-sms.com/\n",
      "http://www.smsyes.com/\n",
      "http://www.smsyes.com/\n",
      "http://www.edihasms.com/\n",
      "http://www.edihasms.com/\n",
      "http://www.smspress.com/\n",
      "http://www.smspress.com/\n",
      "http://www.textmefree.com/\n",
      "http://www.textmefree.com/\n",
      "http://www.cbfsms.com/\n",
      "http://www.cbfsms.com/\n",
      "http://www.hotsms.com/\n",
      "http://www.hotsms.com/\n",
      "http://www.smspup.com/\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android_g1_USSD_codes_to_check_your_cellphone_balance\n",
      "BY_ANDY_THOMPSON By_Kat_Zeman By_Christine_Fenno By_Jonas_Elmerraji\n",
      "USSD codes works in the same way on all networks and GSM phone.\n",
      "The screen will display \"New WAP setting saved successfully.\" If the handset is being configured for the first time, the user will be prompted with \"To activate WAP service, please enter PIN\" when [WAP] is selected.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 sqlite3 tutorial\n",
      "KF_OOE By_Jonas_Elmerraji By_Joseph_Woelfel By_Laurelle_Gilbert\n",
      "You can use the logcat command from your development computer or from a remote adb shell in an emulator/device instance.\n",
      "If you're running logcat from your development computer (versus running it on a remote adb shell), you can also set a default filter expression by exporting a value for the environment variable ANDROID_LOG_TAGS:\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 trick for getting flash updates to properly push to the adp1\n",
      "Just_type_SportsDayDFW.com free_TWoP_toolbar Get_HuffPost_Media Foursquare_revamps\n",
      "Posted on January 16th, 2009 in Android Hacking by Disconnect  Tagged boot, bootloader, development, fastboot, G1 & ADP1, howto, images, root\n",
      "1.fastboot flash:raw boot {kernel} {initrd}\n",
      "Once you have a kernel/ramdisk you like, use [code]fastboot flash:raw boot {kernel} {ramdisk}\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### idea mit thesis-2011-06-12_22-51\n",
      "By_Adam_Ghassemi By_Jonas_Elmerraji DIRECTORS_OF_CAPITAL_CORP.\n",
      "mit algorithm can learn languages\n",
      "1 intentional deception\n",
      "1 process SEC filings, fed meeting & pres release transcripts\n",
      "2) calibrate human readers to do similar things\n",
      "3) compare human and machine reader to statistical \"truth\" measures\n",
      "4) perhaps get human reader data from\n",
      "a analyst stock ratings changes on release of qrtly reports\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Bus stops in banderas bay\n",
      "Queues_spilled Gawking_bystanders_lined Power_outage_darkens Blaze_erupts\n",
      "Oxxo with dozens of bicycles chained to fence at bus stop\n",
      "Car wash at la cruz centro stop\n",
      "near delphin on last paved street before dirtbroad along marina is a nice mexican restaurant--good price on combo plates (80 pesos) for 2 enchiladas, a chimichanga, a taco, beans and rice.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Sunday ToDo\n",
      "KIM_CLIJSTERS_Yeah KF_OOE BY_ANDY_THOMPSON By_Laurelle_Gilbert\n",
      "display pilot chart data in Octave (netcdf) or other GIS program (WorldWind appears to do it)\n",
      "file formats for Roger to be able to read on a Mac\n",
      ".kap and .bsb or .geo and .nos\n",
      "GEO/NOS is a BSB or maptech chart format\n",
      "gdal library reads nos/geo and bsb/kap, but not sure it can write them\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### telephone_long_distance_country_codes_and_mobile_network_access_codes\n",
      "China_Telecom_CHA.NYSE By_Mary_Lennighan CDMA_Handset China_Mobile_CHL.NYSE\n",
      "Greece (GSM1800)\tCellular Operating System Mobile Tel. C-OTE\t202\t01\tn/a\n",
      "Hungary (GSM 1800)\tV.R.A.M. Telecommunications Company\tVODA\t216\t70\tX\t \n",
      "Italy\tTelecom Italia Mobile\tTIM\t222\t01\t \tX\n",
      "Singapore\tSingapore Telecom Mobile \tST GSM\t525\t01\t \tX\n",
      "Taiwan (GSM1800)\tKG Telecom Co. KGT\t466\t88\tX\t \n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 jailbreak for root access\n",
      "Skyhook_SDK By_Jonas_Elmerraji By_Miriam_Reimer World_iNews_Aggregator\n",
      "The ever driven developers over at XDA Developers have devised a method that allows G1 users to update their T-Mobile G1 to the latest Android OS RC30 build while still retaining access to the file-system root.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 instamapper gpstracker instructions, keys, links\n",
      "actually pioneer_LaLanne anyway\n",
      "Best way would be to build an ap that attracts the same people that like gpstracker and then use it to get phone # and key information and send it surreptitiously by e-mail to a central server that could record all this information and \"test\" it.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 activate without a sim card using wifi\n",
      "Foursquare_revamps KF_OOE Windows_logon\n",
      "android g1 activate without a sim card using wifi.\n",
      "2) Use the adb program from the sdk to launch a shell on the phone:\n",
      "5) Reconnect with the adb shell and launch the settings activity (does not require root):\n",
      "6) Using the settings activity that you've launched on the phone's screen, enable wifi.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### You americans you just come2\n",
      "so anyway really THE_PRESIDENT_Yes\n",
      "They care about all these immigrants.\n",
      "Government wont help me when i told them i was having a little financially stressed taking care of them.\n",
      "A big green field and nothing but australians.\n",
      "An older bpudgier australian couple came over to comisserate and tell us this and ask if he was  crazy.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Trip to pv, journal\n",
      "just By_SELENA_ROBERTS\n",
      "They clown in the back of the bus didnt have the floppy hat with a face on the back of it, but his makeup seemed more intricate and elaborate, with sparkles here and there near his eyes and airbrushed highlights of yellow in places.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 android developers community instructions on updating the unlocked android dev phone 1\n",
      "can should do BY_ANDY_THOMPSON\n",
      "As a result, the Market application on such devices is not able to access copy protected apps, whether they are free or paid.\n",
      "Your application will always be accessible to users who have standard configurations though, and if your application (whether it is free or paid) is not copy-protected it will appear on all devices, including developer configurations.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Anna bananas was closing down\n",
      "just BY_ANDY_THOMPSON"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n",
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Christy and davod had been out to san juan island on their 29 ft columbia and decided that \"this sucks.\" So instead of a suitcase full of money and a small boat they got a big boat and no money and set sail anyway.\n",
      "Mikes watched a couple big huricanes come by, \"the bay makes its own weather.\" It pushes hurricanes away.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### jap\n",
      "TOO SHORT to SUMMARIZE\n",
      "MISMATCHED VOCAB\n",
      "jap. \n",
      "‘Š’kŽ–€1 “ú–{lŒ’N¬l”íŒ±ŽÒ‚ð‘ÎÛ‚Æ‚µ‚½‘æI‘ŠŽŽŒ±‚ÅREG25iŽŽŒ ±»Üj‚ÆæsƒoƒCƒIˆã–ò•i‚Å‚ ‚éƒO\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### larissa contact and skype details\n",
      "Visit_www.toastmasters.org Website_www.citizen_news.org By_Dave_Toplikar By_Christina_Stymfal"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "larissa contact and skype details.\n",
      "Larissa contact details in australia for March 2009\n",
      "phone number: \n",
      "skype: larajayne\n",
      "pw: half of her favorite pw, capitalized\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Stock tips\n",
      "QLT_NASDAQ_QLTI lean_hogs_edged March_delivery_HGH9 Premarket_Dow\n",
      "Stock tips\n",
      "Stock tips\n",
      "BMN australian co operating in us mining uranium.\n",
      "2 yr futures contract for 70, extraction cost rose to 41 per lb.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 activate without sim card data plan using root access\n",
      "just KF_OOE SECRETARY_RICE_Yeah\n",
      "The rest of this post will guide you on how to free and activate your Android phone from the T-mobile network in about 5 minutes (assuming you already have the unlock code).\n",
      "From here you can either press menu to add a new APN or edit the settings for the existing one you are viewing.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Huahine\n",
      "just BY_ANDY_THOMPSON\n",
      "Text: Once we found the less crowded anchorage at the southern tip of Huahine, the island became one of our favorite stops so far.\n",
      "To the south of the resort dingy dock is a beautiful grand tree with vertical tall ribbon-like roots that radiate outward from it's base towards the water.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 0 houses in australia\n",
      "Shattered_glass_covered Kluwe_tweets resembling_flower_petals ornately_tiled_floors\n",
      "?, 105 high st, concrete under fence settling and tilted, corner white picket fence, green trees, white weatherboard, stained glassb on all windows, 2 big garages, corner, mntn view deck, new smooth concrete driveway\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 00 Numbers to remember\n",
      "Phyillis_threw By_CITIZEN_STAFF By_Tejas_Gunas DIRECTORS_OF_CAPITAL_CORP.\n",
      "000 Numbers to remember\n",
      "000 Numbers to remember\n",
      "Skype to go in australia\n",
      "Larissas Qld lic no 70140324\n",
      "5 alenola street\n",
      "Chapel hill 4069\n",
      "The point wirless\n",
      "My optus cell phone number\n",
      "Larissa's\n",
      "Our home skype number in florida\n",
      "Imstamapper.com tracking key\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 0 journal--weeds of oz\n",
      "BY_ANDY_THOMPSON anyway\n",
      "Their like a bunch of lemmings in australia, taxed and policed into submission and low level depression.\n",
      "Ian complained about that too and bragged about how he made the cop feel like shit for fining him in a new speed zone trap where theyd reduced the limit.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### malaysian\n",
      "Fold_whipped_cream Serve_chilled Gently_reheat Adjust_seasoning"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Kedai kopi = cafe (typical meal is roti (nan or flat bread) with a reddish brown spicy curry dipping sauce or dahl lentils with carrots and peppers for dipping and a cup of dark chicory coffee supersatured with sugar, sometimes fresh orange or carrot juice served warm from the juicer over a tall glass of ice that chills it within seconds)\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Feeds\n",
      "TOO SHORT to SUMMARIZE\n",
      "Sietsema_Farms feeds Feeders Dow_Jones_Elementized_News"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Feeds. \n",
      "Feeds\n",
      "feeds.wsjonline.com/wsj/podcast_wall_street_journal_tech_news_briefing.xml\n",
      "feeds.wsjonline.com/wsj/podcast_wall_street_journal_tech_news_briefing.xml\n",
      "feeds.wsjonline.com/wsj/podcast_wall\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### idea--business location and competition\n",
      "BY_ANDY_THOMPSON just do KIM_CLIJSTERS_Yeah\n",
      "It makes sense to put you business next to competing businesses if you think you have a competive advantage or discriminator, both in terms of market space, domain name, biz name, and physical retail location.\n",
      "Ensures you make the best mating choice rather than the expeditious (what's that word for taking advantage of a good situation?) one.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### domain names\n",
      "LunchTimeWaster Fridge_Door Customize_Your Randoms\n",
      "domain names.\n",
      "Hookshot\n",
      "Trifecta\n",
      "Hook\n",
      "Score\n",
      "Morethanscore\n",
      "Morescore\n",
      "Fieldplay\n",
      "Fieldhookup\n",
      "Playthefield\n",
      "Ultimateplay\n",
      "Ultumatescore\n",
      "Perfectscore\n",
      "Highscore\n",
      "Scoreme\n",
      "Hookme\n",
      "Hookmeup\n",
      "Thehook\n",
      "Hookup\n",
      "Hookpick\n",
      "Shotpick\n",
      "Assistme\n",
      "Hitme\n",
      "Givengo\n",
      "Assistyou\n",
      "PlayAssist\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Yes optus cell phone mobile\n",
      "website_www.dot.nd.gov ###-###-####_passcode_Seitel banking_centers_#.###.KEY#YOU using_automated_dialers\n",
      "Yes optus cell phone mobile.\n",
      "Yes optus cell phone mobile phone\n",
      "Www.optuszoo.com.au/mobile\n",
      "1831+number -- Temporarily block outgoing caller id for this call\n",
      "1832+number -- temporarily allow outgoing caller id for this call\n",
      "+6141100321 voicemail from cellphone overseas\n",
      "ON or OFF to 159 to turn sms of caller number (voicemail sms)\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 setting up for development\n",
      "SECRETARY_RICE_Yeah KF_OOE By_Kat_Zeman By_Jonas_Elmerraji\n",
      "On the device, go to the home screen, press MENU, select Applications > Development, then enable USB debugging.\n",
      "* If you're developing on 32-bit Windows, you need to install the 32-bit USB driver for adb.\n",
      "* If you're developing on 64-bit Windows Vista, you need to install the 64-bit USB driver for adb.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 adb command reference\n",
      "just BY_ANDY_THOMPSON anyway\n",
      "would issue the install command as soon as the emulator or device instance connected to the adb server, but before the Android system was fully booted, so it would result in an error.\n",
      "Note that ANDROID_LOG_TAGS filter is not exported to the emulator/device instance, if you are running logcat from a remote shell or using adb shell logcat.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Telephone_Dialing_Instructions_for_Mexico\n",
      "referencing_Regal_Beloit ###-###-####_passcode_Seitel jmonteverde@faruqilaw.com_Toll_Free DIG_Propagation_Workshop\n",
      "To dial a Mexican cell phone from the US you must insert another 1 after the country code, e.g. 011-52-1 then the 10 digit number.\n",
      "from Mexico Landline to U.S. or Canada 00 & 1 & Area Code & 7 Digit Phone Number\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### long_distance_country_codes\n",
      "Hutchison_Essar_Telecom TATA_Teleservices Bharti_Hexacom Aircel_Cellular\n",
      "31\tHongkong \tChina Mobile Peoples Telephone Company Ltd.\n",
      "46\tIndia\tBharti Airtel- Andhra Pradesh\t91\n",
      "49\tIndia\tBharti Airtel- Madhya Pradesh\t91\n",
      "51\tIndia\tBharti Airtel- Himachal Pradesh\t91\n",
      "60\tIndia\tIdea Mobile Communications Ltd.\n",
      "68\tIndia\tVodafone Essar Mobile Services Ltd.\n",
      "70\tIndia\tVodafone Essar East Limited (formerly Hutchison Telecom East Ltd.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 (adev1) firmware updates\n",
      "By_Kat_Zeman Visit_www.courant.com_letters txt_file\n",
      "To flash a system image to your device, you need a command-line tool called fastboot and the adb tool provided in the Android SDK.\n",
      "5. Use the adb push command to copy the contents of the radio image package to the update.zip archive on the device's SD card:\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 0 ideas 2009-12\n",
      "do By_Jonas_Elmerraji\n",
      "\"Dont want to keep your prickly toddlers in a pouch-like play pen all day long, give them a puggle.\n",
      "Psychologists rave about the independence and creativity a child can develop if a safety net like a puggle is provided to allow them to safely explore their environment.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 without data plan\n",
      "Visit_www.courant.com_letters Causes_horizontal just\n",
      "9- click the menu button, then select paste,( scroll down to make sure the update.zip file is in there)\n",
      "14- let the update install 100% and read the yellow text, when it says press Home+Back to reboot, Do so.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### telephone keypad cheat sheet for skype\n",
      "phone automated_dialers website_www.dot.nd.gov Sincerely_La_Linea\n",
      "telephone keypad cheat sheet for skype.\n",
      "1=blank\n",
      "2=abc\n",
      "3=def\n",
      "4=ghi\n",
      "5=jkl\n",
      "6=mno\n",
      "7=pqrs\n",
      "8=tuv\n",
      "9=wxyz\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Instamapper GPStracker links, keys, instructions\n",
      "know_MyFace_Spacebook By_Laurelle_Gilbert KF_OOE\n",
      "Links to maps:\n",
      "http://www.instamapper.com/ext?key=16299415166764529961\n",
      "http://mobile.instamapper.com/ext?key=16299415166764529961 \n",
      "To do this, install our application (http://apps.facebook.com/instamapper) and use Facebook tracking key 4135414879490539949.\n",
      "GPS tracking powered by <a href=\"http://www.instamapper.com\">InstaMapper.com</a>\n",
      "These seem like honeypots from google search of \"allinanchor: instamapper-com-ext-key\" or \"allinurl:  \"instamapper com ext key\"\"\n",
      "the honeypot keys seemt o all be even\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 0 tidbits--evolution and dark ages\n",
      "BY_ANDY_THOMPSON pioneer_LaLanne actually that\n",
      "I can pay attention to the categories of connections (when I record or when I read it later) and make a list of those as the first step in creating a language, a notation of thought, insight, knowledge, wisdom...tidbits.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Wifi hotspots in Brisbane \n",
      "v_Myponga v_Yankalilla Little_Malop Gallops_Mar\n",
      "ANZAC Square - Adelaide Street - Brisbane, QLD\n",
      "The Alibi Room - for customers - 720 Brunswick Street Newfarm - 07 3358 6133 - Brisbane\n",
      "State Library of Queensland - Cultural Centre, Stanley Place, South Bank - +61 7 3840 7666 - Brisbane - QLD\n",
      "Afk Cafe - 616 Stanley Street - 07-3846-0445 - Woolloongabba, QLD\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 01 zigbit programming notes\n",
      "KF_OOE By_Christine_Fenno By_Kat_Zeman By_Jonas_Elmerraji\n",
      "Spent an hour debugging hardware issues (wire strands on perf board in harness shorting power and drawing it down to 1.5 or 2V on hw settings tab in studio\n",
      "Clicking the verify button for flash in the jtag ice programme dialog gives the same message, even after properly selecting the hex file.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 multitouch hack\n",
      "KF_OOE BY_ANDY_THOMPSON By_Laurelle_Gilbert By_Jonas_Elmerraji\n",
      "You'll need to type four more commands to mount the file system in a writeable state, change to the system directory, copy the recovery.img file from your SD card to your phone's /system directory, and flash the recovery image, in that order:\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### mexico cell phone service plan details\n",
      "By_Jonas_Elmerraji BY_ANDY_THOMPSON KF_OOE"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Even if your current cell phone company would arrange for your phone calls to your stateside cell phone number to ring your cell phone in Mexico, would you want to pay international long distance charges on every local call that you make or receive?\n",
      "Movistar tarifa Única: This standard plan has both local and long distance calls, to Movistar, other cell phones, and fixed lines at 3.5 pesos per minute.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### hotspot-wdg username and password\n",
      "TOO SHORT to SUMMARIZE\n",
      "Forums_username Forgot_Password WordPress.com_blog DH.N\n",
      "hotspot-wdg username and password. \n",
      "hotspot-wdg username and password: \t\n",
      "CB001945\n",
      "\t810c\n",
      "\t\n",
      "\tto logoff be sure to use:\n",
      "\thttp://192.168.187.1:3990/logoff\n",
      "\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### You americans you just come\n",
      "THE_PRESIDENT_Yes pioneer_LaLanne anyway Echoed_Stoops_That"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.summarization.summarizer:Input text is expected to have at least 10 sentences.\n",
      "WARNING:gensim.summarization.summarizer:Input corpus is expected to have at least 10 documents.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Government wont help me when i told them i was having a little financially stressed taking care of them.\n",
      "An older bpudgier australian couple came over to comisserate and tell us this and ask if he was  crazy.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### email.txt.jap_shifjis\n",
      "TOO SHORT to SUMMARIZE\n",
      "MISMATCHED VOCAB\n",
      "email.txt.jap_shifjis. \n",
      "窶佛�窶冖ﾅｽ窶督坂ぎ1 窶愿ｺ窶怒ﾂ人ﾅ停�卍康ﾂ青ｬﾂ人窶敕ｭﾅ陳ｱﾅｽﾃ停�堙ｰ窶佚篠湘帚�堙�窶堋ｵ窶堋ｽ窶佚ｦ\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### Brisbane\n",
      "Ox_Blackbirds #/#L_#/#L Swamp_Rd Angel_Vaults\n",
      "Option 1 - 45 min Duration 0:45 next 9:18 Walk->Queen St Bus Station B4 235 m 9:22 Bus Route: Brookfield Express 435->Chapel Hill Moggill Rd arr 9:42 9:42 Walk 1415 m\n",
      "Option 3 - 37 min Duration 0:37 prev 9:43 Walk->Queen St Bus Station B4 235 m 9:47 Bus Route: Chapel Hill Express 425->Burbong West arr 10:1 10:11 Walk 595 m\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 00 shopping packing lists\n",
      "Clip_coupons MFI_LN There_Meineke_muffler Mouse_droppings\n",
      "Shopping and packing lists\n",
      "Shopping and packing lists\n",
      "Close and lock hatches and port holes (eng room)\n",
      "Turn up fridge and freezer\n",
      "Eggs in fridge, bread/cereal/flour in freezer\n",
      "Larissa gifts: bike stuff, chocolate, tshirts, gps, running shoes, peeler, toaster\n",
      "For electronics shopping\n",
      "Bike shop\n",
      "12 v bug zapper or fly paper\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### 0 Numbers to remember\n",
      "number_+#.###.###.####_Passcode Loto_Québec_Banco By_CITIZEN_STAFF By_Tejas_Gunas\n",
      "0 Numbers to remember\n",
      "0 Numbers to remember\n",
      "Skype to go in australia\n",
      "The point wirless\n",
      "My optus cell phone number\n",
      "Larissa's\n",
      "Our home skype number in florida\n",
      "Imstamapper.com tracking key\n",
      "Larissas Qld lic no 70140324\n",
      "5 alenola street\n",
      "Chapel hill 4069\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "### android g1 tips\n",
      "Make_ajc.com KF_OOE Foursquare_revamps Just_type_SportsDayDFW.com\n",
      "Bonus Tip: If you want to add in another caller while you’re on the phone, press the Send key while in a call.\n",
      "Navigate to the home screen and select Menu > Settings > SD card & phone storage.\n",
      "3) Using the $25/month G1 internet service from t-mobile it will connect to google and activate the phone.\n"
     ]
    }
   ],
   "source": [
    "for meta in tree:\n",
    "    if meta['name'].lower().startswith('stuff'):\n",
    "        continue\n",
    "    print('')\n",
    "    print('-'*80)\n",
    "    print('### ' + os.path.splitext(meta['name'])[0])\n",
    "    text = os.path.splitext(meta['name'])[0] + '. \\n' + open(meta['path']).read()\n",
    "    try:\n",
    "        summary = summarize(text, word_count=50)\n",
    "    except ValueError:\n",
    "        print('TOO SHORT to SUMMARIZE')\n",
    "        summary = text[:200]\n",
    "    try:\n",
    "        print(' '.join(pair[0] for pair in wv.most_similar(positive=[w for w in summary.split() if w in wv.vocab])[1:5] if '%' not in pair[0])) \n",
    "    except ValueError:\n",
    "        print('MISMATCHED VOCAB')\n",
    "    print(summary)\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
