{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from gatenlp import Document\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets1 = []\n",
    "n = 0\n",
    "with open(\"../../weverify-tweet-similarity/data/raw/plandemic01Mar31Oct_inclusive.json\", \"rt\", encoding=\"utf-8\") as infp:\n",
    "    for line in infp:\n",
    "        n += 1\n",
    "        tweets1.append(line)\n",
    "        if n >= 100:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'conversation_id_str,coordinates,created_at,datetimestamp,essid,favorite_count,forward_pivot,full_text,geo,id_str,in_reply_to_screen_name,in_reply_to_status_id_str,in_reply_to_user_id_str,lang,limited_actions,name,place,quote_count,quoted_status_id_str,reply_count,retweet_count,screen_name,source,user_id_str,wit'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\",\".join(sorted(list(json.loads(tweets1[0]).keys())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>#KXMHLSRRXA-wrapper { color: black !important; }</style>\n",
       "<div id=\"KXMHLSRRXA-wrapper\">\n",
       "\n",
       "<div>\n",
       "<style>\n",
       "#KXMHLSRRXA-content {\n",
       "    width: 100%;\n",
       "    height: 100%;\n",
       "    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-row {\n",
       "    width: 100%;\n",
       "    display: flex;\n",
       "    flex-direction: row;\n",
       "    flex-wrap: nowrap;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-col {\n",
       "    border: 1px solid grey;\n",
       "    display: inline-block;\n",
       "    min-width: 200px;\n",
       "    padding: 5px;\n",
       "    /* white-space: normal; */\n",
       "    /* white-space: pre-wrap; */\n",
       "    overflow-y: auto;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-hdr {\n",
       "    font-size: 1.2rem;\n",
       "    font-weight: bold;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-label {\n",
       "    margin-bottom: -15px;\n",
       "    display: block;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-input {\n",
       "    vertical-align: middle;\n",
       "    position: relative;\n",
       "    *overflow: hidden;\n",
       "}\n",
       "\n",
       "#KXMHLSRRXA-popup {\n",
       "    display: none;\n",
       "    color: black;\n",
       "    position: absolute;\n",
       "    margin-top: 10%;\n",
       "    margin-left: 10%;\n",
       "    background: #aaaaaa;\n",
       "    width: 60%;\n",
       "    height: 60%;\n",
       "    z-index: 50;\n",
       "    padding: 25px 25px 25px;\n",
       "    border: 1px solid black;\n",
       "    overflow: auto;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-selection {\n",
       "    margin-bottom: 5px;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-featuretable {\n",
       "    margin-top: 10px;\n",
       "}\n",
       "\n",
       ".KXMHLSRRXA-fname {\n",
       "    text-align: left !important;\n",
       "    font-weight: bold;\n",
       "    margin-right: 10px;\n",
       "}\n",
       ".KXMHLSRRXA-fvalue {\n",
       "    text-align: left !important;\n",
       "}\n",
       "</style>\n",
       "  <div id=\"KXMHLSRRXA-content\">\n",
       "        <div id=\"KXMHLSRRXA-popup\" style=\"display: none;\">\n",
       "        </div>\n",
       "        <div class=\"KXMHLSRRXA-row\" id=\"KXMHLSRRXA-row1\" style=\"max-height: 20em; min-height:5em;\">\n",
       "            <div id=\"KXMHLSRRXA-text-wrapper\" class=\"KXMHLSRRXA-col\" style=\"width:70%;\">\n",
       "                <div class=\"KXMHLSRRXA-hdr\" id=\"KXMHLSRRXA-dochdr\"></div>\n",
       "                <div id=\"KXMHLSRRXA-text\">\n",
       "                </div>\n",
       "            </div>\n",
       "            <div id=\"KXMHLSRRXA-chooser\" class=\"KXMHLSRRXA-col\" style=\"width:30%; border-left-width: 0px;\"></div>\n",
       "        </div>\n",
       "        <div class=\"KXMHLSRRXA-row\" id=\"KXMHLSRRXA-row2\" style=\"max-height: 14em; min-height: 3em;\">\n",
       "            <div id=\"KXMHLSRRXA-details\" class=\"KXMHLSRRXA-col\" style=\"width:100%; border-top-width: 0px;\">\n",
       "            </div>\n",
       "        </div>\n",
       "    </div>\n",
       "    \n",
       "    <script type=\"application/json\" id=\"KXMHLSRRXA-data\">\n",
       "    {\"annotation_sets\": {\"Original markups\": {\"name\": \"detached-from:Original markups\", \"annotations\": [{\"type\": \"Tweet\", \"start\": 0, \"end\": 113, \"id\": 0, \"features\": {\"id_str\": \"1238967569647693825\", \"quote_count\": 0, \"retweet_count\": 0, \"lang\": \"en\"}}], \"next_annid\": 1}}, \"text\": \"Wake up Kiwi's. There is far more to this pan demic than meets the eye.\\n\\nThe Plandemic.  \\nhttps://t.co/e7FsvL2sm7\", \"features\": {}, \"offset_type\": \"j\", \"name\": \"\"}\n",
       "    </script>\n",
       "    <script type=\"text/javascript\">\n",
       "        gatenlp_run(\"KXMHLSRRXA-\");\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "</div></div>"
      ],
      "text/plain": [
       "Document(Wake up Kiwi's. There is far more to this pan demic than meets the eye.\n",
       "\n",
       "The Plandemic.  \n",
       "https://t.co/e7FsvL2sm7,features=Features({}),anns={'Original markups': AnnotationSet([Annotation(0,113,Tweet,features=Features({'id_str': '1238967569647693825', 'quote_count': 0, 'retweet_count': 0, 'lang': 'en'}),id=0)])})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Document.load_mem(tweets1[1], fmt=\"tweet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (gatenlp-36)",
   "language": "python",
   "name": "gatenlp-36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
