{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Clustering polls into topics\n",
    "> Using titles/descriptions of polls for clustering. Goal is unsupervised grouping into topicsish based on [this](https://radimrehurek.com/gensim/auto_examples/core/run_topics_and_transformations.html#sphx-glr-auto-examples-core-run-topics-and-transformations-py)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Note: You may need to run `python -m spacy download de_core_news_sm`, if not already done, to process the German language."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pprint\n",
    "\n",
    "import pandas as pd\n",
    "import polars as pl\n",
    "from bundestag.fine_logging import setup_logging\n",
    "import logging\n",
    "from bundestag.paths import get_paths\n",
    "from bundestag.data.transform.abgeordnetenwatch.transform import get_polls_parquet_path\n",
    "from bundestag.ml.poll_clustering import (\n",
    "    SpacyTransformer,\n",
    "    clean_text,\n",
    "    compare_word_frequencies,\n",
    ")\n",
    "from gensim.models.coherencemodel import CoherenceModel\n",
    "from plotnine import (\n",
    "    aes,\n",
    "    geom_histogram,\n",
    "    ggplot,\n",
    "    labs,\n",
    "    scale_fill_manual,\n",
    "    geom_line,\n",
    "    geom_area,\n",
    "    geom_bar,\n",
    "    geom_point,\n",
    "    scale_x_continuous,\n",
    ")\n",
    "from functools import partial\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "setup_logging(logging.INFO)\n",
    "\n",
    "paths = get_paths(\"../data\")\n",
    "paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "legislature_ids = [67, 83, 97, 111, 132, 161]\n",
    "_df_polls = []\n",
    "for legislature_id in legislature_ids:\n",
    "    file = get_polls_parquet_path(legislature_id, paths.preprocessed_abgeordnetenwatch)\n",
    "    tmp = pl.read_parquet(file)\n",
    "    _df_polls.append(tmp)\n",
    "df_polls = pl.concat(_df_polls)\n",
    "df_polls.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polls = df_polls.with_columns(pl.col(\"poll_date\").str.to_date())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering based on poll title\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sanity checking word counts, longest and shortest titles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleaning using spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://towardsdatascience.com/end-to-end-topic-modeling-in-python-latent-dirichlet-allocation-lda-35ce4ed6b3e0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = \"poll_title\"\n",
    "nlp_col = f\"{col}_nlp_processed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "st = SpacyTransformer()\n",
    "df_polls = df_polls.with_columns(\n",
    "    **{\n",
    "        nlp_col: pl.col(col).map_elements(\n",
    "            partial(clean_text, nlp=st.nlp), return_dtype=pl.List(pl.String)\n",
    "        )\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polls.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspecting word frequencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_word_frequencies(df_polls, col, nlp_col)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The word count distribution shifted to lower values, as could be expected, but no documents were left without any words."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transforming using LDA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's first find a suitable `num_topics` value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_topics_grid = [5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]\n",
    "\n",
    "_stats = []\n",
    "for num_topics in num_topics_grid:\n",
    "    st.fit_lda(df_polls[nlp_col].to_list(), num_topics=num_topics)\n",
    "    # compute metrics - https://radimrehurek.com/gensim/models/coherencemodel.html\n",
    "    cm = CoherenceModel(model=st.lda_model, corpus=st.corpus, coherence=\"u_mass\")\n",
    "    coherence = cm.get_coherence()\n",
    "\n",
    "    log_perplexity = st.lda_model.log_perplexity(st.corpus)\n",
    "    _stats.append(\n",
    "        {\n",
    "            \"num topics\": num_topics,\n",
    "            \"coherence\": coherence,\n",
    "            \"log perplexity\": log_perplexity,\n",
    "        }\n",
    "    )\n",
    "\n",
    "stats = pl.from_dicts(_stats)\n",
    "stats = stats.unpivot(index=\"num topics\", variable_name=\"metric\", value_name=\"value\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Visualize (coherence spike and perplexity drop indicates an interesting number of topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(stats, aes(\"num topics\", \"value\", color=\"metric\"))\n",
    "    + geom_line()\n",
    "    + scale_x_continuous(breaks=num_topics_grid)\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit again with the chosen `num_topics` value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_topics = 8\n",
    "st.fit_lda(df_polls[nlp_col].to_list(), num_topics=num_topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_polls = st.transform(df_polls, col=nlp_col)\n",
    "df_polls.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = df_polls.select([\"poll_date\", \"poll_id\"] + st.nlp_cols).unpivot(\n",
    "    index=[\"poll_date\", \"poll_id\"], value_name=\"weight\", variable_name=\"topic\"\n",
    ")\n",
    "\n",
    "tmp = (\n",
    "    tmp.group_by([\"poll_date\", \"topic\"])\n",
    "    .agg(**{\"n polls\": pl.col(\"poll_id\").n_unique(), \"weight\": pl.col(\"weight\").sum()})\n",
    "    .with_columns(**{\"normalized weight\": pl.col(\"weight\") / pl.col(\"n polls\")})\n",
    ")\n",
    "tmp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(tmp, aes(x=\"poll_date\", y=\"normalized weight\", color=\"topic\", fill=\"topic\"))\n",
    "    + geom_point()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = df_polls.select([\"poll_date\", \"poll_id\"] + st.nlp_cols).unpivot(\n",
    "    index=[\"poll_date\", \"poll_id\"], value_name=\"weight\", variable_name=\"topic\"\n",
    ")\n",
    "\n",
    "tmp = (\n",
    "    tmp.group_by([pl.col(\"poll_date\").dt.year(), \"topic\"])\n",
    "    .agg(**{\"n polls\": pl.col(\"poll_id\").n_unique(), \"weight\": pl.col(\"weight\").sum()})\n",
    "    .with_columns(**{\"normalized weight\": pl.col(\"weight\") / pl.col(\"n polls\")})\n",
    ")\n",
    "tmp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(\n",
    "    ggplot(tmp, aes(x=\"poll_date\", y=\"normalized weight\", color=\"topic\", fill=\"topic\"))\n",
    "    + geom_area()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Discovered topics:\")\n",
    "pprint.pprint(st.lda_topics)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {
    "height": "220px",
    "width": "310px"
   },
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
