{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division, print_function, absolute_import\n",
    "\n",
    "import os\n",
    "from decimal import Decimal\n",
    "from traceback import print_exc\n",
    "\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "\n",
    "# you really want to be efficient about RAM, so user iter and itertools\n",
    "# from itertools import izip\n",
    "from twip.constant import DATA_PATH\n",
    "\n",
    "import matplotlib\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading tweets (could take a minute or so)...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "# this should load 100k tweets in about a minute\n",
    "print('Loading tweets (could take a minute or so)...')\n",
    "df = pd.read_csv(os.path.join(DATA_PATH, 'all_tweets.csv'), encoding='utf-8', engine='python')\n",
    "# in iPython Notebook print out df.columns to show that many of them contain dots\n",
    "# rename the columns to be attribute-name friendly\n",
    "df.columns = [label.replace('.', '_') for label in df.columns]\n",
    "print('Done.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The raw table shape is (99535, 350)\n",
      "After dropping columns with fewer than 330 nonnull values, the table shape is (99535, 246)\n",
      "After dropping rows with fewer than 50 nonnull values, the table shape is (98405, 246)\n",
      "Of the 246 columns, 8 are actually DataFrames\n",
      "Column quoted_status_id is a 2-wide DataFrame\n",
      "Finished converting column quoted_status_id to type <class 'pandas.core.series.Series'>(float64)\n",
      "Column quoted_status_id_str is a 2-wide DataFrame\n",
      "Finished converting column quoted_status_id_str to type <class 'pandas.core.series.Series'>(float64)\n",
      "Column retweeted_status_quoted_status_id is a 2-wide DataFrame\n",
      "Finished converting column retweeted_status_quoted_status_id to type <class 'pandas.core.series.Series'>(float64)\n",
      "Column retweeted_status_quoted_status_id_str is a 2-wide DataFrame\n",
      "Finished converting column retweeted_status_quoted_status_id_str to type <class 'pandas.core.series.Series'>(float64)\n",
      "Of the 242 columns, 0 are still DataFrames after trying to convert both columns to long integers\n"
     ]
    }
   ],
   "source": [
    "# in iPython Notebook, try dropping with lower thresholds, checking column and row count each time\n",
    "print('The raw table shape is {}'.format(df.shape))\n",
    "nonnull_rows = 330\n",
    "nonnull_cols = 50\n",
    "df = df.dropna(axis=1, thresh=nonnull_rows)\n",
    "print('After dropping columns with fewer than {} nonnull values, the table shape is {}'.format(nonnull_rows, df.shape))\n",
    "df = df.dropna(axis=0, thresh=nonnull_cols)\n",
    "print('After dropping rows with fewer than {} nonnull values, the table shape is {}'.format(nonnull_cols, df.shape))\n",
    "\n",
    "\n",
    "# in ipython notebook, explore and describe the DataFrame columns\n",
    "print('Of the {} columns, {} are actually DataFrames'.format(len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))\n",
    "# remove dataframes with only 2 columns and one is the _str of the other:\n",
    "for col in df.columns:\n",
    "    if isinstance(df[col], pd.DataFrame):\n",
    "        print('Column {} is a {}-wide DataFrame'.format(col, len(df[col].columns)))\n",
    "        if df[col].columns[1] == df[col].columns[0] + '_str':\n",
    "            print('Column {} looks easy because it has sub-columns {}'.format(col, df[col].columns))\n",
    "            df[col] = df[col][df[col].columns[1]]\n",
    "        else:\n",
    "            try:\n",
    "                assert(float(df[col].iloc[:, 0].max()) == float(df[col].iloc[:, 1].max()))\n",
    "                df[col] = df[col].fillna(-1, inplace=False)\n",
    "                series = pd.Series([int(Decimal(x)) for x in df[col].iloc[:, 1].values]).astype('int64').copy()\n",
    "                del df[col]\n",
    "                df[col] = series\n",
    "                print('Finished converting column {} to type {}({})'.format(col, type(df[col]), df[col].dtype))\n",
    "            except:\n",
    "                print_exc()\n",
    "\n",
    "print('Of the {} columns, {} are still DataFrames after trying to convert both columns to long integers'.format(\n",
    "    len(df.columns), sum([not isinstance(df[col], pd.Series) for col in df.columns])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df.describe() stats:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/numpy/lib/function_base.py:3823: RuntimeWarning: Invalid value encountered in percentile\n",
      "  RuntimeWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "quoted_status_retweet_count (float64)\n",
      "count      754.000000\n",
      "mean       225.071618\n",
      "std       1466.800347\n",
      "min          0.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      23318.000000\n",
      "Name: quoted_status_retweet_count, dtype: float64\n",
      "\n",
      "quoted_status_user_favourites_count (float64)\n",
      "count       754.000000\n",
      "mean       8663.230769\n",
      "std       24785.825849\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      269482.000000\n",
      "Name: quoted_status_user_favourites_count, dtype: float64\n",
      "\n",
      "quoted_status_user_followers_count (float64)\n",
      "count    7.540000e+02\n",
      "mean     2.342662e+05\n",
      "std      1.687835e+06\n",
      "min      1.200000e+01\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      3.850663e+07\n",
      "Name: quoted_status_user_followers_count, dtype: float64\n",
      "\n",
      "quoted_status_user_friends_count (float64)\n",
      "count       754.000000\n",
      "mean       3775.824934\n",
      "std       14528.856528\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      181261.000000\n",
      "Name: quoted_status_user_friends_count, dtype: float64\n",
      "\n",
      "quoted_status_user_id (float64)\n",
      "count    7.540000e+02\n",
      "mean     2.545139e+16\n",
      "std      1.321662e+17\n",
      "min      4.816000e+03\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.268294e+17\n",
      "Name: quoted_status_user_id, dtype: float64\n",
      "\n",
      "quoted_status_user_id_str (float64)\n",
      "count    7.540000e+02\n",
      "mean     2.545139e+16\n",
      "std      1.321662e+17\n",
      "min      4.816000e+03\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.268294e+17\n",
      "Name: quoted_status_user_id_str, dtype: float64\n",
      "\n",
      "quoted_status_user_listed_count (float64)\n",
      "count       754.000000\n",
      "mean       1639.852785\n",
      "std        8547.734310\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      173929.000000\n",
      "Name: quoted_status_user_listed_count, dtype: float64\n",
      "\n",
      "quoted_status_user_statuses_count (float64)\n",
      "count       754.000000\n",
      "mean      25134.769231\n",
      "std       49857.465868\n",
      "min           4.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      354746.000000\n",
      "Name: quoted_status_user_statuses_count, dtype: float64\n",
      "\n",
      "quoted_status_user_utc_offset (float64)\n",
      "count      623.000000\n",
      "mean     -8774.638844\n",
      "std      17871.374986\n",
      "min     -39600.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      43200.000000\n",
      "Name: quoted_status_user_utc_offset, dtype: float64\n",
      "\n",
      "retweet_count (float64)\n",
      "count     98405.000000\n",
      "mean         56.837823\n",
      "std         965.450259\n",
      "min           0.000000\n",
      "25%           0.000000\n",
      "50%           1.000000\n",
      "75%           4.000000\n",
      "max      166648.000000\n",
      "Name: retweet_count, dtype: float64\n",
      "\n",
      "retweeted_status_favorite_count (float64)\n",
      "count     34564.000000\n",
      "mean        164.355630\n",
      "std        1510.055532\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      144713.000000\n",
      "Name: retweeted_status_favorite_count, dtype: float64\n",
      "\n",
      "retweeted_status_id (float64)\n",
      "count    3.456400e+04\n",
      "mean     7.211856e+17\n",
      "std      3.109526e+16\n",
      "min      3.613619e+09\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275290e+17\n",
      "Name: retweeted_status_id, dtype: float64\n",
      "\n",
      "retweeted_status_id_str (float64)\n",
      "count    3.456400e+04\n",
      "mean     7.211856e+17\n",
      "std      3.109526e+16\n",
      "min      3.613619e+09\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275290e+17\n",
      "Name: retweeted_status_id_str, dtype: float64\n",
      "\n",
      "retweeted_status_in_reply_to_status_id (float64)\n",
      "count    1.117000e+03\n",
      "mean     7.204485e+17\n",
      "std      2.992915e+16\n",
      "min      3.613582e+09\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275190e+17\n",
      "Name: retweeted_status_in_reply_to_status_id, dtype: float64\n",
      "\n",
      "retweeted_status_in_reply_to_status_id_str (float64)\n",
      "count    1.117000e+03\n",
      "mean     7.204485e+17\n",
      "std      2.992915e+16\n",
      "min      3.613582e+09\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275190e+17\n",
      "Name: retweeted_status_in_reply_to_status_id_str, dtype: float64\n",
      "\n",
      "retweeted_status_in_reply_to_user_id (float64)\n",
      "count    1.393000e+03\n",
      "mean     1.438282e+16\n",
      "std      1.004640e+17\n",
      "min      5.339000e+03\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.234400e+17\n",
      "Name: retweeted_status_in_reply_to_user_id, dtype: float64\n",
      "\n",
      "retweeted_status_in_reply_to_user_id_str (float64)\n",
      "count    1.393000e+03\n",
      "mean     1.438282e+16\n",
      "std      1.004640e+17\n",
      "min      5.339000e+03\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.234400e+17\n",
      "Name: retweeted_status_in_reply_to_user_id_str, dtype: float64\n",
      "\n",
      "retweeted_status_quoted_status_favorite_count (float64)\n",
      "count     1412.000000\n",
      "mean       128.676346\n",
      "std        641.991896\n",
      "min          0.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      22699.000000\n",
      "Name: retweeted_status_quoted_status_favorite_count, dtype: float64\n",
      "\n",
      "retweeted_status_quoted_status_retweet_count (float64)\n",
      "count     1412.000000\n",
      "mean       105.660765\n",
      "std        484.172127\n",
      "min          0.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      17293.000000\n",
      "Name: retweeted_status_quoted_status_retweet_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_favourites_count (float64)\n",
      "count     34564.000000\n",
      "mean       6556.545828\n",
      "std       19014.864620\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      424498.000000\n",
      "Name: retweeted_status_user_favourites_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_followers_count (float64)\n",
      "count    3.456400e+04\n",
      "mean     1.658826e+05\n",
      "std      1.035896e+06\n",
      "min      0.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      3.850754e+07\n",
      "Name: retweeted_status_user_followers_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_friends_count (float64)\n",
      "count     34564.000000\n",
      "mean       6972.357424\n",
      "std       20664.927254\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      554702.000000\n",
      "Name: retweeted_status_user_friends_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_id (float64)\n",
      "count    3.456400e+04\n",
      "mean     2.692105e+16\n",
      "std      1.360227e+17\n",
      "min      2.200000e+01\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.267090e+17\n",
      "Name: retweeted_status_user_id, dtype: float64\n",
      "\n",
      "retweeted_status_user_id_str (float64)\n",
      "count    3.456400e+04\n",
      "mean     2.692105e+16\n",
      "std      1.360227e+17\n",
      "min      2.200000e+01\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.267090e+17\n",
      "Name: retweeted_status_user_id_str, dtype: float64\n",
      "\n",
      "retweeted_status_user_listed_count (float64)\n",
      "count     34564.000000\n",
      "mean       1364.608871\n",
      "std        6070.081716\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      173930.000000\n",
      "Name: retweeted_status_user_listed_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_statuses_count (float64)\n",
      "count    3.455200e+04\n",
      "mean     2.550541e+04\n",
      "std      5.831330e+04\n",
      "min      1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      1.235370e+06\n",
      "Name: retweeted_status_user_statuses_count, dtype: float64\n",
      "\n",
      "retweeted_status_user_utc_offset (float64)\n",
      "count    25303.000000\n",
      "mean     -9104.821563\n",
      "std      15862.625806\n",
      "min     -39600.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      46800.000000\n",
      "Name: retweeted_status_user_utc_offset, dtype: float64\n",
      "\n",
      "user_favourites_count (float64)\n",
      "count     98391.000000\n",
      "mean       2610.526115\n",
      "std       11279.405726\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      661637.000000\n",
      "Name: user_favourites_count, dtype: float64\n",
      "\n",
      "user_followers_count (float64)\n",
      "count    9.839100e+04\n",
      "mean     3.093464e+03\n",
      "std      6.197171e+04\n",
      "min      0.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      1.038394e+07\n",
      "Name: user_followers_count, dtype: float64\n",
      "\n",
      "user_friends_count (float64)\n",
      "count     98391.000000\n",
      "mean       1291.734925\n",
      "std        5653.808701\n",
      "min           0.000000\n",
      "25%                NaN\n",
      "50%                NaN\n",
      "75%                NaN\n",
      "max      382464.000000\n",
      "Name: user_friends_count, dtype: float64\n",
      "\n",
      "user_id (float64)\n",
      "count    9.839100e+04\n",
      "mean     6.717982e+16\n",
      "std      2.086599e+17\n",
      "min      2.200000e+01\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.274945e+17\n",
      "Name: user_id, dtype: float64\n",
      "\n",
      "user_id_str (float64)\n",
      "count    9.839100e+04\n",
      "mean     6.717982e+16\n",
      "std      2.086599e+17\n",
      "min      2.200000e+01\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.274945e+17\n",
      "Name: user_id_str, dtype: float64\n",
      "\n",
      "user_listed_count (float64)\n",
      "count    98391.000000\n",
      "mean       349.854540\n",
      "std       1053.931396\n",
      "min          0.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      95363.000000\n",
      "Name: user_listed_count, dtype: float64\n",
      "\n",
      "user_statuses_count (float64)\n",
      "count    9.839100e+04\n",
      "mean     6.382632e+04\n",
      "std      1.374776e+05\n",
      "min      1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      2.357119e+06\n",
      "Name: user_statuses_count, dtype: float64\n",
      "\n",
      "user_utc_offset (float64)\n",
      "count    57744.000000\n",
      "mean     -5419.622818\n",
      "std      18536.802288\n",
      "min     -39600.000000\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      46800.000000\n",
      "Name: user_utc_offset, dtype: float64\n",
      "\n",
      "quoted_status_id (float64)\n",
      "count    9.728300e+04\n",
      "mean     5.595990e+15\n",
      "std      6.336782e+16\n",
      "min     -1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275211e+17\n",
      "Name: quoted_status_id, dtype: float64\n",
      "\n",
      "quoted_status_id_str (float64)\n",
      "count    9.728300e+04\n",
      "mean     5.595990e+15\n",
      "std      6.336782e+16\n",
      "min     -1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275211e+17\n",
      "Name: quoted_status_id_str, dtype: float64\n",
      "\n",
      "retweeted_status_quoted_status_id (float64)\n",
      "count    9.728300e+04\n",
      "mean     1.054961e+16\n",
      "std      8.674438e+16\n",
      "min     -1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275131e+17\n",
      "Name: retweeted_status_quoted_status_id, dtype: float64\n",
      "\n",
      "retweeted_status_quoted_status_id_str (float64)\n",
      "count    9.728300e+04\n",
      "mean     1.054961e+16\n",
      "std      8.674438e+16\n",
      "min     -1.000000e+00\n",
      "25%               NaN\n",
      "50%               NaN\n",
      "75%               NaN\n",
      "max      7.275131e+17\n",
      "Name: retweeted_status_quoted_status_id_str, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('df.describe() stats:')\n",
    "desc = df.describe()\n",
    "for col, stats in desc.T.iterrows():\n",
    "    print('')\n",
    "    print('{} ({})'.format(col, df[col].dtype if isinstance(df[col], pd.Series) else type(df[col])))\n",
    "    print(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to compute a ProfileReport, including correlation between columns, skew etc\n",
      "{'total_missing': 0.63156452411853847, 'UNIQUE': 0, 'CONST': 3, 'nvar': 243, 'REJECTED': 15, 'n': 98405, 'NUM': 28, 'CORR': 12, 'memsize': '182.4 MiB', 'DATE': 0, 'CAT': 200, 'recordsize': '1.9 KiB', 'n_duplicates': 0}\n"
     ]
    }
   ],
   "source": [
    "# this takes a few minutes\n",
    "print('Using pandas_profiling to generate more detailed stats, including correlation between columns, skew etc')\n",
    "# pandas_profiling.ProfileReport raises Tkinter exceptions before it can produce any output,\n",
    "#  at least describe produces a dataframe of stats\n",
    "report = pandas_profiling.describe(df)\n",
    "print(report['table'])\n",
    "\n",
    "print('')\n",
    "for col, stats in report['variables'].iterrows():\n",
    "    print('')\n",
    "    print(col)\n",
    "    # print('{} ({})'.format(col, df[col].dtype if isinstance(df[col], pd.Series) else type(df[col])))\n",
    "    print(stats)\n",
    "\n",
    "# and if you thought that was bad, try printing out all the report['freq'] dict of histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "UnicodeEncodeError",
     "evalue": "'ascii' codec can't encode character u'\\u2026' in position 213: ordinal not in range(128)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mUnicodeEncodeError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-6bfc6e9c7e63>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprofile\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpandas_profiling\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mProfileReport\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/pandas_profiling/__init__.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, df)\u001b[0m\n\u001b[0;32m     15\u001b[0m         \u001b[0mdescription_set\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdescribe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m         self.html = to_html(df.head(),\n\u001b[1;32m---> 17\u001b[1;33m                             description_set)\n\u001b[0m\u001b[0;32m     18\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mto_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moutputfile\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mDEFAULT_OUTPUTFILE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/pandas_profiling/base.pyc\u001b[0m in \u001b[0;36mto_html\u001b[1;34m(sample_df, stats_object)\u001b[0m\n\u001b[0;32m    302\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    303\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcol\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[1;32min\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miteritems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 304\u001b[1;33m             \u001b[0mformatted_values\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfmt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    305\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    306\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrow\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m&\u001b[0m \u001b[0msix\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mviewkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrow_formatters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/hobs/.virtualenvs/twip/local/lib/python2.7/site-packages/pandas_profiling/base.pyc\u001b[0m in \u001b[0;36mfmt\u001b[1;34m(value, name)\u001b[0m\n\u001b[0;32m    246\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mvalue_formatters\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mformatters\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mDEFAULT_FLOAT_FORMATTER\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    247\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 248\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    249\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    250\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfreq_table\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfreqtable\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtable_template\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrow_template\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_number_of_items_in_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mUnicodeEncodeError\u001b[0m: 'ascii' codec can't encode character u'\\u2026' in position 213: ordinal not in range(128)"
     ]
    }
   ],
   "source": [
    "profile = pandas_profiling.ProfileReport(df)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11+"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
