{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "98289ad1-a2f9-4a95-8106-2b3c970681e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "src_path = os.path.join(os.path.dirname(os.getcwd()), 'src')\n",
    "if src_path not in sys.path:\n",
    "    sys.path.append(os.path.join(os.path.dirname(os.getcwd()), 'src'))\n",
    "\n",
    "import kg, utils\n",
    "from dotenv import load_dotenv\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "24ef8aec-b2d4-4659-aa59-5125995b23c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen = kg.KnowledgeGraphGenerator(model = \"openai/gpt-4o\", api_key=os.getenv(\"OPENAI_API_KEY\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5defce43-75ef-440b-b268-517a94c3f4b4",
   "metadata": {},
   "source": [
    "# Manual Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f1aa580d-bf7d-4db8-82e3-bca875ac347c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#file_name = \"BenGurionFirstPassageWiki\"\n",
    "#file_name = \"JerusalemWikiShort\"\n",
    "#file_name = \"Ramban19_2\"\n",
    "#file_name = \"IbnShuib19_2\"\n",
    "#file_name = \"MeiratEinayim19_2\"\n",
    "#file_name = \"KeterShemTov19_2\"\n",
    "#file_name = \"Eskorial19_2\"\n",
    "#file_name = \"Vatikan214_19_2\"\n",
    "#file_name = \"London487_19_2\"\n",
    "file_name = \"zfad\"\n",
    "chunk_size = 300\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "01fb2a13-b681-497b-904f-145f4dd9581f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#path = \"../../Ramban/\"\n",
    "path = \"/Users/hadarmiller/Downloads/Eliezer/\"\n",
    "with open(f\"{path}txt/{file_name}.txt\", \"r\", encoding=\"utf-8\") as file:\n",
    "    text = file.read()\n",
    "    text = text.strip().replace(\"\\n\", \" \")\n",
    "\n",
    "#text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b0bd39cc-877a-418d-9a10-c457ec6cddbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#text.split(\".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "728ea690-402b-49d5-824e-dd0870499efd",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.extractConcepts(text=text, chunk_size=chunk_size, verbose=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0d735747-c6a3-4e9d-9d89-8badb441543d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a924821c-6ec2-416b-b662-5a5ff4e45e7f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dropped non-English predicate: חיבר\n",
      "Number of RDF relations: 125\n"
     ]
    }
   ],
   "source": [
    "kggen.extractRelations(verbose = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b9577407-9d75-473d-86f3-36016b81f2d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd35cc2a-8904-42c3-98fa-f04ed0513bdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Round 1] Grouping 159 concepts in parallel batches...\n",
      "[Round 1] Reduced to 141 concepts.\n",
      "[Round 2] Grouping 141 concepts in parallel batches...\n",
      "[Round 2] Reduced to 140 concepts.\n",
      "[Round 3] Grouping 140 concepts in parallel batches...\n",
      "[Round 3] Reduced to 139 concepts.\n",
      "Reached maximum iterations (3). Stopping early.\n",
      "Number of concepts after grouping: 139\n"
     ]
    }
   ],
   "source": [
    "kggen.groupConcepts(chunk_size= 160, threshold= 160, max_iterations= 2, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "95a434c2-eb05-41f8-8288-6a8554629776",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "960bc619-f276-478e-8d16-0165be4d4274",
   "metadata": {},
   "outputs": [],
   "source": [
    "#recommendation = kggen.ontologyRecommendation()\n",
    "#print(recommendation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aefb4689-c419-4d51-a21f-ddf17f535581",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.relations2ontology([\"SKOS\"]) # \"DUBLIN_CORE\", \"SKOS\", \"CDOCCRM\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "91e9eb3b-2238-4e41-b1de-cbd5c2cbca2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.relations2ontology([\"DUBLIN_CORE\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "22411bb4-b1f9-4ea3-9f4c-148528d0cf8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.relations2ontology([\"CDOCCRM\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c36a2133-ef3c-4b13-bdbb-8a8c1c5a44a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.relations2ontology([\"LLM\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "db359b6f-ab81-4270-8fb1-e889fa1284f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.relations2ontology([\"CDOCCRM\", \"SKOS\", \"DUBLIN_CORE\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5bb6dffd-0f42-4b55-ad61-79c200cebb60",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.ontologies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "08179e5c-fe78-4cd1-9ada-8eea889994da",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.dump_graph(f\"{path}json/{file_name}_c{chunk_size}.json\")\n",
    "#kggen.dump_graph(f\"/Users/hadarmiller/Downloads/Ramban/json/JerOnePassage.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "7ce37d73-bd13-4a92-a094-94abe383aef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.init_graph(path=f\"{path}json/{file_name}_c{chunk_size}.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "98b64116-59e5-4e88-909a-6a71643c972c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ontology = \"MIX\"\n",
    "kggen.graph2Ontology(ontology)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "aa68461a-5729-4a0e-922d-9d3553ca5823",
   "metadata": {},
   "outputs": [],
   "source": [
    "viz = kggen.visualize(f\"{path}vis/{file_name}_c{chunk_size}_ontology_{ontology}.html\")\n",
    "#viz = kggen.visualize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba7d60fa-5dc3-4f23-bc9b-6228380b0672",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bd42a44a-4dd7-480d-91e2-2e076df2775a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "results.html\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"100%\"\n",
       "            height=\"800px\"\n",
       "            src=\"results.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x15ce33220>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "viz = kggen.visualize()\n",
    "viz.show(\"results.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3af43930-3852-4b0b-b214-eb24f6d657b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "7f04a3fc-93d5-4c4c-b33f-f99d27bf8769",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.concepts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca4cd20d-366d-4189-baf6-68fb54d2ff5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The term in Entities: True\n",
      "The term in Relations: True\n",
      "The term in Concepts: True\n",
      "The term in Concepts ALTs: False\n"
     ]
    }
   ],
   "source": [
    "term = \"כרובים על הפרכת\"\n",
    "term = \"קדש הקדשים\"\n",
    "tmp_rel = [x[0] for x in kggen.graph.relations] + [x[2] for x in kggen.graph.relations]\n",
    "cmp_conc = [x.prefLabel_he for x in kggen.graph.concepts]\n",
    "tmp_con_alt = []\n",
    "for c in kggen.graph.concepts:\n",
    "    tmp_con_alt += c.altLabels_he\n",
    "print(f\"The term in Entities: {term in kggen.graph.entities}\\nThe term in Relations: {term in tmp_rel}\\nThe term in Concepts: {term in cmp_conc}\")\n",
    "print(f\"The term in Concepts ALTs: {term in tmp_con_alt}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "429048d7-abd0-459d-ac96-b36aaa709b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "def mergeNodesEdges(prefLabel: str, altLabels: list[str]):\n",
    "\n",
    "    relations = list(kggen.graph.relations)\n",
    "    # Filter out relations where subject or object is in altLabels\n",
    "    # and replace them with the prefLabel.\n",
    "    # This ensures that all relations involving altLabels are replaced with the prefLabel.\n",
    "    # This is done to unify the entities under a single concept.\n",
    "    new_relations = [x for x in relations if x[0] not in  altLabels and x[2] not in altLabels]\n",
    "\n",
    "    for al in altLabels:\n",
    "        new_relations.extend([(prefLabel,r,s) for o,r,s in relations if o==al])\n",
    "        new_relations.extend([(o,r,prefLabel) for o,r,s in relations if s==al])\n",
    "    \n",
    "\n",
    "    new_nodes = [x for x in kggen.graph.entities if x not in altLabels]\n",
    "\n",
    "    return new_relations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "454b3699-f6b2-4175-9b4e-fc6953f3ed8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "tmp = kggen.graph.concepts[2]\n",
    "\n",
    "rel_tmp = []\n",
    "for r in kggen.graph.relations:\n",
    "    rel_tmp.append(r[0])\n",
    "    rel_tmp.append(r[2])\n",
    "print(term in rel_tmp)\n",
    "\n",
    "ttt = mergeNodesEdges(tmp.prefLabel_he, tmp.altLabels_he)\n",
    "tmp_con_alt2 = []\n",
    "for c in ttt:\n",
    "    tmp_con_alt2.append(c[0])\n",
    "    tmp_con_alt2.append(c[2])\n",
    "print(term in tmp_con_alt2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6191e0d4-70a3-41ac-81fb-9ce18e7a99d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kggen.graph.concepts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6edd3815-33e5-4536-8b9b-32d57195a93b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Full Process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29e37763-7fa2-4ce4-a4bd-d55e74016b0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#file_name = \"JerusalemWikiShort\"\n",
    "#file_name = \"BenGurionFirstPassageWiki\"\n",
    "#file_name = \"Ramban19_2\"\n",
    "file_name = \"IbnShuib19_2\"\n",
    "#file_name = \"MeiratEinayim19_2\" #ERROR\n",
    "#file_name = \"KeterShemTov19_2\"\n",
    "#file_name = \"Eskorial19_2\"\n",
    "#file_name = \"Vatikan214_19_2\"\n",
    "#file_name = \"London487_19_2\"\n",
    "\n",
    "chunk_size = 300\n",
    "text_path = \"../../Ramban/txt/\"\n",
    "text_file_name = file_name + \".txt\"\n",
    "ontologies = [\"SKOS\", \"DUBLIN_CORE\", \"CDOCCRM\", \"LLM\"]\n",
    "viz_path = \"../../Ramban/vis/\"\n",
    "\n",
    "verbose = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0def64a1-5043-4424-a8ed-f412bf3b2566",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of subgraphs: 9\n",
      "Number of Entities: 62\n",
      "Entities list: כרוביו, מקדש, תבנית, פניהם איש אל אחיו, העדות, משכן, אלהי ישראל, עשר מנורות, ארון ברית ה', עצי הלבנון, כרובים, פרק המוכר פירות, מרכבה, תורה שבכתב, הרב ז\"ל, המעלה, הבנין, ייחוד, הארון, הלוחות, נושאי הכבוד, נהר כבר, ארון הברית, הכרובים, חיה, חמש אמות, עשר כיורים, תפארת, שפע היורד, מורי נר\"ו, חמש האחד, עליונים, תבנית לכרובים העליונים, חמש משמאל, בנין, זהב, כבוד, חמש מימין, עין בעין, חמש כנגד חמש, הכפורת, העניינים הרוחנים, היחוד, מכונות, שם הגדול, יושב הכרובים, רצון השלם, אברהם, מרכבת יחזקאל, מורי י\"א, כנף האחד, דו פרצופים, שפע, הצואה, צדק ושלום נשקו, מקשה, כסא כבוד, החיה, תורה שבע\"פ, גבוהים, יחזקאל, שלמה המלך\n",
      "Number of subgraphs: 9\n",
      "Number of RDF relations: 47\n",
      "validateRelations-->Found invalid relations after merging concepts: {'מקדש', 'תבנית', 'עשר מנורות', 'העדות', 'פניהם איש אל אחיו', 'אלהי ישראל', 'עצי הלבנון', 'כרובים', 'מרכבה', 'תורה שבכתב', 'הרב ז\"ל', 'הבנין', 'ייחוד', 'הלוחות', 'נושאי הכבוד', 'נהר כבר', 'ארון הברית', 'עשר כיורים', 'מורי נר\"ו', 'עליונים', 'זהב', 'חמש כנגד חמש', 'הכפורת', 'מכונות', 'העניינים הרוחנים', 'שם הגדול', 'יושב הכרובים', 'רצון השלם', 'מורי י\"א', 'כנף האחד', 'שפע', 'צדק ושלום נשקו', 'הצואה', 'מקשה', 'כסא כבוד', 'החיה', 'תורה שבע\"פ', 'יחזקאל', 'שלמה המלך'}\n",
      "Number of concepts: 44\n",
      "Concepts list: כרובים, מקדש, ארון הברית, תורה שבכתב, תורה שבע\"פ, מרכבה, שפע, יחזקאל, שלמה המלך, עצי הלבנון, הכפורת, הלוחות, העדות, החיה, הבנין, תפארת, עליונים, הרב ז\"ל, מורי נר\"ו, מורי י\"א, נהר כבר, עשר מנורות, עשר כיורים, מכונות, חמש כנגד חמש, פניהם איש אל אחיו, צדק ושלום נשקו, מקשה, כסא כבוד, דו פרצופים, הצואה, העניינים הרוחנים, ייחוד, שם הגדול, פרק המוכר פירות, אברהם, המעלה, תבנית, כנף האחד, זהב, אלהי ישראל, נושאי הכבוד, עין בעין, רצון השלם\n",
      "Ontology SKOS: prepared\n",
      "Ontology DUBLIN_CORE: prepared\n",
      "Ontology CDOCCRM: prepared\n",
      "Ontology LLM: prepared\n",
      "Ontology SKOS: converted to graph relations\n",
      "Error converting ontology SKOS to graph: Error generating HTML file with pyvis: non existent node 'יושב הכרובים'\n",
      "Ontology DUBLIN_CORE: converted to graph relations\n",
      "Error converting ontology DUBLIN_CORE to graph: Error generating HTML file with pyvis: non existent node 'יושב הכרובים'\n",
      "Ontology CDOCCRM: converted to graph relations\n",
      "Error converting ontology CDOCCRM to graph: Error generating HTML file with pyvis: non existent node 'יושב הכרובים'\n",
      "Ontology LLM: converted to graph relations\n",
      "Error converting ontology LLM to graph: Error generating HTML file with pyvis: non existent node 'יושב הכרובים'\n"
     ]
    }
   ],
   "source": [
    "kggen.generateKG(text_path, text_file_name, ontologies, viz_path, chunk_size, verbose)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f5072ff7-7882-4bbb-9400-d0279b2178f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "kggen.dump_graph(f\"../../Ramban/json/{file_name}_c{chunk_size}.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fcf73e0-0ea4-429c-9327-b3322cd9d674",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02514a55-6685-4e10-b4f3-3ef74589d591",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kggen2",
   "language": "python",
   "name": "kggen2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
