{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tidyX examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install tidyX==1.6.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: tidyX\n",
      "Version: 1.6.6\n",
      "Summary: Python package to clean raw tweets for ML applications\n",
      "Home-page: \n",
      "Author: Lucas Gómez Tobón, Jose Fernando Barrera\n",
      "Author-email: lucasgomeztobon@gmail.com, jf.barrera10@uniandes.edu.co\n",
      "License: MIT\n",
      "Location: c:\\users\\lucas\\anaconda3\\envs\\bx\\lib\\site-packages\n",
      "Requires: emoji, nltk, numpy, pandas, regex, spacy, thefuzz, Unidecode\n",
      "Required-by: \n"
     ]
    }
   ],
   "source": [
    "!pip show tidyX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tidyX import TextPreprocessor as tp\n",
    "from tidyX import TextNormalization as tn\n",
    "from tidyX import TextVisualizer as tv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stemming and Lemmatizing Texts Efficiently"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `stemmer()` and `lemmatizer()` functions each accept a single token as input. Thus, if we aim to normalize an entire text or a corpus, we would need to iterate over each token in the string using these functions. This approach might be inefficient, especially if the input contains repeated words.\n",
    "\n",
    "This tutorial demonstrates how to utilize the `unnest_tokens()` function to apply normalization functions just once for every unique word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tp' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\Lucas\\Documents\\Tidytweets\\docs\\source\\tutorials\\tutorial.ipynb Cell 7\u001b[0m line \u001b[0;36m1\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/Lucas/Documents/Tidytweets/docs/source/tutorials/tutorial.ipynb#Y165sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m tp\u001b[39m.\u001b[39mload_data(file \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mspanish\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'tp' is not defined"
     ]
    }
   ],
   "source": [
    "tp.load_data(file = \"spanish\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tutorial: Word Cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from wordcloud import WordCloud\n",
    "import matplotlib.pyplot as plt\n",
    "import spacy\n",
    " \n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = pd.read_excel(r\"../../../data/Tweets sobre venezuela.xlsx\")\n",
    "tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine all documents into a single string\n",
    "text = \" \".join(doc for doc in tweets['Snippet'])\n",
    "\n",
    "# Generate a word cloud image\n",
    "wordcloud = WordCloud(background_color = \"white\", width = 800, height = 400).generate(text)\n",
    "\n",
    "# Display the generated image\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.title(\"WordCloud before tidyX\")\n",
    "plt.imshow(wordcloud, interpolation = 'bilinear')\n",
    "plt.axis(\"off\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets['clean'] = tweets['Snippet'].apply(lambda x: tp.preprocess(x, delete_emojis = False, extract = False,\n",
    "                                                                  remove_stopwords = True))\n",
    "tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_df = tp.unnest_tokens(df = tweets.copy(), input_column = \"clean\", id_col = None, unique = True)\n",
    "token_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load spacy's model\n",
    "model = spacy.load('es_core_news_lg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply spanish_lemmatizer function to lemmatize the token\n",
    "token_df[\"lemma\"] = token_df[\"clean\"].apply(lambda x: tn.lemmatizer(token = x, model = model))\n",
    "token_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_df[\"lemma\"] = token_df[\"lemma\"].apply(lambda x: tp.remove_words(x, remove_stopwords = True))\n",
    "token_df = token_df[[\"clean\", \"lemma\"]]\n",
    "token_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_long = tp.unnest_tokens(df = tweets.copy(), input_column = \"clean\", id_col = None, unique = False)\n",
    "tweets_long "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_clean2 = tweets_long.merge(token_df, how = \"left\", on = \"clean\").groupby([\"Snippet\", \"id\"])[\"lemma\"].agg(lambda x: \" \".join(x)).reset_index()\n",
    "tweets_clean2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_clean2['lemma'] = tweets_clean2['lemma'].apply(lambda x: tp.remove_extra_spaces(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Combine all documents into a single string\n",
    "text = \" \".join(doc for doc in tweets_clean2['lemma'])\n",
    "\n",
    "# Generate a word cloud image\n",
    "wordcloud = WordCloud(background_color = \"white\", width = 800, height = 400).generate(text)\n",
    "\n",
    "# Display the generated image\n",
    "plt.figure(figsize=(10, 5))\n",
    "plt.title(\"WordCloud after tidyX\")\n",
    "plt.imshow(wordcloud, interpolation = 'bilinear')\n",
    "plt.axis(\"off\");"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "BX",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
