{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nibelungenlied in Referenzkorpus  Mittelhochdeutsch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import collections\n",
    "from lxml import etree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The corpus that we use here may be found at `link`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "link = \"https://www.linguistics.rub.de/rem/access/index.html\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Unzip this and inside the directory, you will see `filename`: this is the annotated Nibelungenlied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"M321-G1.xml\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We prepare our parser."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "stringify = etree.XPath(\"string()\")\n",
    "parser = etree.XMLParser(load_dtd=True, no_network=False)\n",
    "tree = etree.parse(os.path.join(filename), parser=parser)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see what's inside it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<text id=\"M321\">\\n  <header>\\n    <text>Nibelungenlied</text>\\n    <abbr_ddd>Nib</abbr_ddd>\\n    <abbr_m'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root = tree.getroot()\n",
    "decoded_root = etree.tostring(root).decode(\"utf-8\")\n",
    "decoded_root[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "That's an XML file! We can extract contents with XPath selector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_annotations(entry):\n",
    "    return {child.tag: child.get(\"tag\") for child in entry.getchildren()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = [extract_annotations(entry) for entry in root.findall(\".//tok_anno\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_by_tag(tag, tokens):\n",
    "    return [token[tag] for token in tokens if tag in token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_text = extract_by_tag(\"norm\", tokens)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lemmata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmata = extract_by_tag(\"lemma\", tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter_nibelungenlied = collections.Counter(lemmata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dër', 1536),\n",
       " ('ër', 976),\n",
       " ('ich', 282),\n",
       " ('dô', 274),\n",
       " ('sîn', 256),\n",
       " ('vil(e)', 248),\n",
       " ('unte', 247),\n",
       " ('von', 197),\n",
       " ('haben', 191),\n",
       " ('wësen', 189),\n",
       " ('ir', 184),\n",
       " ('ze', 175),\n",
       " ('mit', 168),\n",
       " ('dazz', 166),\n",
       " ('ir(e)', 139),\n",
       " ('sô', 136),\n",
       " ('in', 131),\n",
       " ('dâr', 127),\n",
       " ('sprëchen', 117),\n",
       " ('küni(n)g', 113)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter_nibelungenlied.most_common(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmata_set = set(lemmata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15073, 1374)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lemmata), len(lemmata_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "normalized_to_lemma = {token[\"norm\"]: token[\"lemma\"] for token in tokens if \"lemma\" in token}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemma_to_normalized = {lemma: {token[\"norm\"] for token in tokens \n",
    "                               if \"norm\" in token and \"lemma\" in token and token[\"lemma\"] == lemma}\n",
    "                       for lemma in lemmata_set}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Nibelunc', 'Nibelunge', 'Nibelungen', 'Nibelunges'}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemma_to_normalized[\"Nibelung\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Pars oratori*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_tags = extract_by_tag(\"pos\", tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_set = set(pos_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18499, 58)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pos_tags), len(pos_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_to_lemmata = {pos: {token[\"lemma\"] for token in tokens \n",
    "                               if \"lemma\" in token and \"pos\" in token and token[\"pos\"] == pos}\n",
    "                       for pos in pos_set}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'$_'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(list(pos_set))[0] # originale Interpunktion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"$_\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ADJA', 'ADJD', 'ADJS', 'ADJN']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos for pos in pos_set if pos.startswith(\"ADJ\")] # Adjectives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pos_to_lemmata[\"ADJA\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'hundert', 'lëben', 'tûsent', 'wërben'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"ADJS\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'all-wâre',\n",
       " 'arm',\n",
       " 'be-kant',\n",
       " 'be-rèit(e)',\n",
       " 'be-rèitet',\n",
       " 'blôz',\n",
       " 'brèit',\n",
       " 'er-kant',\n",
       " 'ge-hazz',\n",
       " 'ge-muot',\n",
       " 'ge-mèit',\n",
       " 'ge-nædig-lich',\n",
       " 'ge-rëht',\n",
       " 'ge-sunt',\n",
       " 'ge-sëzzen',\n",
       " 'ge-triuwe',\n",
       " 'ge-tân',\n",
       " 'ge-waltig',\n",
       " 'ge-wiss',\n",
       " 'grimm(e)',\n",
       " 'grème-lich',\n",
       " 'grôz',\n",
       " 'guot',\n",
       " 'gërn(e)',\n",
       " 'hold',\n",
       " 'hêr-lich',\n",
       " 'hövisch',\n",
       " 'kund',\n",
       " 'künde',\n",
       " 'küène',\n",
       " 'lang',\n",
       " 'lièb',\n",
       " 'lièht',\n",
       " 'lobe-lich',\n",
       " 'lære',\n",
       " 'lèid',\n",
       " 'michel',\n",
       " 'milte',\n",
       " 'nazz',\n",
       " 'niuwe',\n",
       " 'offen',\n",
       " 'rëht',\n",
       " 'rîche',\n",
       " 'rôt',\n",
       " 'scharpf',\n",
       " 'schoène',\n",
       " 'schuldig',\n",
       " 'schîn',\n",
       " 'sippe',\n",
       " 'stark',\n",
       " 'swære',\n",
       " 'sælig',\n",
       " 'tiur(e)',\n",
       " 'tièf',\n",
       " 'trûrig',\n",
       " 'trüèbe',\n",
       " 'tôt',\n",
       " 'un-be-kant',\n",
       " 'un-ge-bunten',\n",
       " 'un-ge-logen',\n",
       " 'un-ge-schèiden',\n",
       " 'un-ge-tèilet',\n",
       " 'un-künde',\n",
       " 'un-müge-lich',\n",
       " 'un-müèzig',\n",
       " 'un-ver-daget',\n",
       " 'un-ver-diènet',\n",
       " 'un-ver-zaget',\n",
       " 'unter-tân',\n",
       " 'vroè-lich',\n",
       " 'vrum',\n",
       " 'vrèm(e)de',\n",
       " 'vrô',\n",
       " 'wille-komen',\n",
       " 'wît',\n",
       " 'zornig',\n",
       " 'èdel(e)',\n",
       " 'èl-lènte',\n",
       " 'übel',\n",
       " 'über-müète'}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"ADJD\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bald',\n",
       " 'bi-dèrbe',\n",
       " 'brèit',\n",
       " 'ge-mèit',\n",
       " 'ge-tân',\n",
       " 'grôz',\n",
       " 'guot',\n",
       " 'hêr(e)',\n",
       " 'hôh',\n",
       " 'jung',\n",
       " 'küène',\n",
       " 'lang',\n",
       " 'lièht-ge-varw',\n",
       " 'lobe-bære',\n",
       " 'lobe-lich',\n",
       " 'lobe-sam',\n",
       " 'mære',\n",
       " 'niuwe',\n",
       " 'rîche',\n",
       " 'rôt',\n",
       " 'schoène',\n",
       " 'snëll',\n",
       " 'stark',\n",
       " 'swind(e)',\n",
       " 'tièf',\n",
       " 'vrèm(e)de',\n",
       " 'vèste',\n",
       " 'wilde',\n",
       " 'wît',\n",
       " 'èdel(e)',\n",
       " 'ûz-er-kor(e)n'}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"ADJN\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['APPR']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos for pos in pos_set if pos.startswith(\"AP\")] # Prepositions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'abe',\n",
       " 'after',\n",
       " 'ane',\n",
       " 'bî',\n",
       " 'durh',\n",
       " 'gègen',\n",
       " 'hinter',\n",
       " 'in',\n",
       " 'mit',\n",
       " 'mit-same(n)t',\n",
       " 'nâh',\n",
       " 'umbe',\n",
       " 'unter',\n",
       " 'von',\n",
       " 'vor(e)',\n",
       " 'vür(e)',\n",
       " 'wider',\n",
       " 'ze',\n",
       " 'zuo-ze',\n",
       " 'zwischen',\n",
       " 'âne',\n",
       " 'ûf',\n",
       " 'ûz',\n",
       " 'ûzer',\n",
       " 'über'}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"APPR\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['AVW', 'AVG', 'AVD']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos for pos in pos_set if pos.startswith(\"AV\")] # Adverbs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# [pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"AV\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'drî',\n",
       "  'siben',\n",
       "  'siben-zëhen',\n",
       "  'vièr',\n",
       "  'zwèin-zig',\n",
       "  'zwèl(i)v',\n",
       "  'zwêne',\n",
       "  'èin'},\n",
       " {'vièr', 'zwèl(i)v', 'zwêne', 'èin'}]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Numbers\n",
    "[pos_to_lemmata[pos] for pos in pos_set if pos.startswith(\"CAR\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DDS', 'DDA', 'DDN', 'DDART']"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# definite determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DD\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dër', 'sëlb'}, {'dise', 'jèner', 'so-l(i)ch', 'sëlb'}, {'sëlb'}, {'dër'}]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ pos_to_lemmata[pos] for pos in [pos for pos in [pos for pos in pos_set if pos.startswith(\"DD\")]]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DGA']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# general determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DG\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'swèl(i)ch', 'swëder'}]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"DG\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DIN', 'DIA', 'DID', 'DIART', 'DIS']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Indefinite determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DI\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'all', 'bèide'},\n",
       " {'all',\n",
       "  'all-so-l(i)ch',\n",
       "  'bèide',\n",
       "  'de-wëder',\n",
       "  'deh-èin',\n",
       "  'iè-mann',\n",
       "  'ièd-wëder',\n",
       "  'iètes-lich',\n",
       "  'manig',\n",
       "  'niè-mann',\n",
       "  'èin',\n",
       "  'èin-ander',\n",
       "  'ëte(s)-lich'},\n",
       " {'bèide'},\n",
       " {'èin'},\n",
       " {'all', 'ander', 'bèide', 'deh-èin', 'iètes-lich', 'manig'}]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"DI\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DPOSN', 'DPOSA', 'DPOSS']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# possessive determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DP\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'dîn', 'mîn', 'sîn'},\n",
       " {'dîn', 'ir(e)', 'iuwer', 'mîn', 'sîn', 'unser'},\n",
       " {'mîn', 'sîn'}]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[ pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"DP\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DRELS']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# relative determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DR\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'dër'}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"DRELS\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# interrogative determinants\n",
    "[pos for pos in pos_set if pos.startswith(\"DW\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# foreign words\n",
    "[pos for pos in pos_set if pos.startswith(\"FM\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ITJ']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Interjection\n",
    "[pos for pos in pos_set if pos.startswith(\"ITJ\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ach', 'hèi', 'wâfen', 'wê', 'ô'}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_to_lemmata[\"ITJ\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['KO*', 'KON', 'KOUS', 'KOKOM']"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Conjunctions\n",
    "[pos for pos in pos_set if pos.startswith(\"KO\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'all-sam(e)',\n",
       "  'all-sô',\n",
       "  'sam(e)',\n",
       "  'swanne~dazz',\n",
       "  'sît~dazz',\n",
       "  'sô',\n",
       "  'wante',\n",
       "  'wëder',\n",
       "  'ê~dazz'},\n",
       " {'bèide', 'dës', 'noh', 'oder', 'unte'},\n",
       " {'danne',\n",
       "  'dazz',\n",
       "  'diè~wîle',\n",
       "  'durh~dazz',\n",
       "  'dô',\n",
       "  'nû',\n",
       "  'obe',\n",
       "  'sîd',\n",
       "  'unz(e)',\n",
       "  'unz(e)~dazz',\n",
       "  'wan',\n",
       "  'wan~dazz',\n",
       "  'êr'},\n",
       " {'danne', 'wan'}]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"KO\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NE', 'NA']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Nouns\n",
    "[pos for pos in pos_set if pos.startswith(\"N\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'all', 'bèide'},\n",
       " {'all',\n",
       "  'all-so-l(i)ch',\n",
       "  'bèide',\n",
       "  'de-wëder',\n",
       "  'deh-èin',\n",
       "  'iè-mann',\n",
       "  'ièd-wëder',\n",
       "  'iètes-lich',\n",
       "  'manig',\n",
       "  'niè-mann',\n",
       "  'èin',\n",
       "  'èin-ander',\n",
       "  'ëte(s)-lich'},\n",
       " {'bèide'},\n",
       " {'èin'},\n",
       " {'all', 'ander', 'bèide', 'deh-èin', 'iètes-lich', 'manig'}]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"DI\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PAVG', 'PAVAP', 'PAVD', 'PAVW']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pronominal adverbs\n",
    "[pos for pos in pos_set if pos.startswith(\"PA\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'swâr/+nâh'},\n",
       " {'ane/dâr+',\n",
       "  'ane/dâr.+',\n",
       "  'bî/dâr+',\n",
       "  'hèime/dâr+',\n",
       "  'inne/dâr+',\n",
       "  'inne/hièr+',\n",
       "  'mit(e)/dâr+',\n",
       "  'mit(e)/wâr+',\n",
       "  'nâh/dâr+',\n",
       "  'nâh/swâr+',\n",
       "  'umbe/dâr+',\n",
       "  'umbe/wâr+',\n",
       "  'unter/dâr+',\n",
       "  'unter/dâr.+',\n",
       "  'von(e)/dâr+',\n",
       "  'von(e)/wâr+',\n",
       "  'vor(e)/dâr+',\n",
       "  'wider(e)/dâr+',\n",
       "  'zuo/dâr+',\n",
       "  'în/dâr+',\n",
       "  'über(e)/dâr+'},\n",
       " {'dâr/+ane',\n",
       "  'dâr/+bî',\n",
       "  'dâr/+hèime',\n",
       "  'dâr/+inne',\n",
       "  'dâr/+mit(e)',\n",
       "  'dâr/+nâh',\n",
       "  'dâr/+umbe',\n",
       "  'dâr/+unter',\n",
       "  'dâr/+von(e)',\n",
       "  'dâr/+vor(e)',\n",
       "  'dâr/+wider(e)',\n",
       "  'dâr/+zuo',\n",
       "  'dâr/+în',\n",
       "  'dâr/+über(e)',\n",
       "  'dâr/.+ane',\n",
       "  'dâr/.+unter.+',\n",
       "  'hièr/+inne'},\n",
       " {'wâr/+mit(e)', 'wâr/+umbe', 'wâr/+von(e)'}]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [pos for pos in pos_set if pos.startswith(\"PA\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PI', 'PW', 'PRF', 'PPER', 'PG']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Pronouns\n",
    "[pos for pos in pos_set \n",
    " if pos in [\"PG\", \"PI\", \"PPER\", \"PRF\", \"PW\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'swër'},\n",
       " {'man'},\n",
       " {'dû', 'ich', 'ir', 'wir', 'ër'},\n",
       " {'dû', 'ich', 'ir', 'sich', 'ër'},\n",
       " {'wër'}]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in [\"PG\", \"PI\", \"PPER\", \"PRF\", \"PW\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['PTKVZ', 'PTKNEG', 'PTK', 'PTKANT', 'PTKA']"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Particles\n",
    "[pos for pos in pos_set if pos.startswith(\"PT\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'abe/+lâzen',\n",
       "  'abe/+slahen',\n",
       "  'ane/+bièten',\n",
       "  'ane/+er-dwingen',\n",
       "  'ane/+ge-vâhen',\n",
       "  'ane/+ge-winnen',\n",
       "  'ane/+hèben',\n",
       "  'ane/+kapfen',\n",
       "  'ane/+loufen',\n",
       "  'ane/+ruofen',\n",
       "  'ane/+schiffen',\n",
       "  'bî/+wësen',\n",
       "  'innen/+bringen',\n",
       "  'mit(e)/+volgen',\n",
       "  'nider(e)/+gân',\n",
       "  'nider(e)/+rîten',\n",
       "  'nider(e)/+vallen',\n",
       "  'vür(e)/+sènten',\n",
       "  'wider(e)/+ge-winnen',\n",
       "  'wider(e)/+gëben',\n",
       "  'wider(e)/+slahen',\n",
       "  'ûf/+blicken',\n",
       "  'ûf/+rihten',\n",
       "  'ûf/+spannen',\n",
       "  'ûf/+tuon',\n",
       "  'ûf/+îlen',\n",
       "  'ûz/+rîten/+gân'},\n",
       " {'ne', 'niht'},\n",
       " {'sô'},\n",
       " {'jâ'},\n",
       " {'ze'}]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in \n",
    " [pos for pos in pos_set if pos.startswith(\"PT\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['VAIMP', 'VAFIN', 'VAPP', 'VAINF']"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Auxiliary verbs\n",
    "[pos for pos in pos_set if pos.startswith(\"VA\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'sîn'},\n",
       " {'haben', 'sîn', 'wërden', 'wësen', 'wësen/bî+'},\n",
       " {'wërden', 'wësen'},\n",
       " {'haben', 'sîn', 'wërden', 'wësen'}]"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in\n",
    " [pos for pos in pos_set if pos.startswith(\"VA\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['VMINF', 'VMFIN']"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# modal verbs\n",
    "[pos for pos in pos_set if pos.startswith(\"VM\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'wèllen'},\n",
       " {'durfen',\n",
       "  'ge-turren',\n",
       "  'kunnen',\n",
       "  'mügen',\n",
       "  'müèzen',\n",
       "  'sol(e)n',\n",
       "  'turren',\n",
       "  'wèllen'}]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[pos_to_lemmata[pos] for pos in \n",
    " [pos for pos in pos_set if pos.startswith(\"VM\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['VVFIN', 'VVPP', 'VVPS', 'VVINF', 'VVIMP']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Complete verbs\n",
    "[pos for pos in pos_set if pos.startswith(\"VV\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# [pos_to_lemmata[pos] for pos in \n",
    "#  [pos for pos in pos_set if pos.startswith(\"VV\")]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Morphology"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "inflections = extract_by_tag(\"infl\", tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15073, 283)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(inflections), len(set(inflections))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "inflection_class = extract_by_tag(\"inflClass\", tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15073, 40)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(inflection_class), len(set(inflection_class))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
