{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# opengrid imports\n",
    "from opengrid.library import misc, houseprint, caching, analysis\n",
    "from opengrid import config\n",
    "c=config.Config()\n",
    "\n",
    "# other imports\n",
    "import pandas as pd\n",
    "import charts\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "\n",
    "# configuration for the plots\n",
    "DEV = c.get('env', 'type') == 'dev' # DEV is True if we are in development environment, False if on the droplet\n",
    "\n",
    "if not DEV:\n",
    "    # production environment: don't try to display plots\n",
    "    import matplotlib\n",
    "    matplotlib.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.dates import HourLocator, DateFormatter, AutoDateLocator, num2date\n",
    "\n",
    "if DEV:\n",
    "    if c.get('env', 'plots') == 'inline':\n",
    "        %matplotlib inline\n",
    "    else:\n",
    "        %matplotlib qt\n",
    "else:\n",
    "    pass # don't try to render plots\n",
    "plt.rcParams['figure.figsize'] = 12,8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hp = houseprint.Houseprint()\n",
    "sensors = hp.get_sensors(sensortype='electricity') # sensor objects\n",
    "\n",
    "# Remove some sensors\n",
    "exclude = [\n",
    "            '565de0a7dc64d8370aa321491217b85f' # 3E\n",
    "          ]\n",
    "solar = [x.key for x in hp.search_sensors(type='electricity', system='solar')]\n",
    "exclude += solar\n",
    "\n",
    "for s in sensors:\n",
    "    if s.key in exclude:\n",
    "        sensors.remove(s)\n",
    "\n",
    "hp.init_tmpo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#hp.sync_tmpos()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# The first time, this will take a very looong time to get all the detailed data for building the cache\n",
    "# Afterwards, this is quick\n",
    "caching.cache_results(hp=hp, sensors=sensors, function='daily_min', resultname='elec_daily_min')\n",
    "caching.cache_results(hp=hp, sensors=sensors, function='daily_max', resultname='elec_daily_max')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cache_min = caching.Cache(variable='elec_daily_min')\n",
    "cache_max = caching.Cache(variable='elec_daily_max')\n",
    "dfdaymin = cache_min.get(sensors=sensors)\n",
    "dfdaymax = cache_max.get(sensors=sensors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    sensor = hp.search_sensors(key='3aa4')[0]\n",
    "    df = sensor.get_data(head=pd.Timestamp('20151117'), tail=pd.Timestamp('20160104'))\n",
    "    charts.plot(df, stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pygal\n",
    "from IPython.display import SVG, HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "html_pygal = \"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html>\n",
    "  <head>\n",
    "  <script type=\"text/javascript\" src=\"http://kozea.github.com/pygal.js/javascripts/svg.jquery.js\"></script>\n",
    "  <script type=\"text/javascript\" src=\"http://kozea.github.com/pygal.js/javascripts/pygal-tooltips.js\"></script>\n",
    "    <!-- ... -->\n",
    "  </head>\n",
    "  <body>\n",
    "    <figure>\n",
    "      {chart}\n",
    "    </figure>\n",
    "  </body>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "html2=\"\"\"\n",
    "<!DOCTYPE html>\n",
    "<html lang=\"en\">\n",
    "<meta charset=\"utf-8\"/>\n",
    "\n",
    "<div>{chart}|safe</div>\n",
    "</html>\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def chart_timeseries(df, title=None):\n",
    "    # I think there is a bug in pygal: datetimes are converted to UTC based on the current locale\n",
    "    # So I have to add the difference between local time and UTC\n",
    "    df.index += pd.Timedelta(seconds=3600)\n",
    "\n",
    "    custom_style = pygal.style.Style(background='transparent')\n",
    "    datetimeline = pygal.DateTimeLine(width=600, height=350, explicit_size=True,\n",
    "                                      x_label_rotation=35, truncate_label=-1,\n",
    "                                      x_value_formatter=lambda dt: dt.strftime('%d/%m %H:%M:%S (UTC)'),\n",
    "                                      show_legend=False,\n",
    "                                      style=custom_style)\n",
    "    for col in df:\n",
    "        datetimeline.add(col, [(i, df.ix[i,col]) for i in df.index])\n",
    "\n",
    "    datetimeline.title = title\n",
    "    chart =  datetimeline.render(is_unicode=True)\n",
    "    return chart"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chart = chart_timeseries(df.dropna().ix[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "HTML(html2.format(chart=chart))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "chart"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next plot shows that some periods are missing.  Due to the cumulative nature of the electricity counter, we still have the total consumption.  However, it is spread out of the entire period.  So we don't know the standby power during these days, and we have to remove those days.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Clean out the data: \n",
    "# First remove days with too low values to be realistic\n",
    "dfdaymin[dfdaymin < 10] = np.nan\n",
    "# Now remove days where the minimum=maximum (within 1 Watt difference)\n",
    "dfdaymin[(dfdaymax - dfdaymin) < 1] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    charts.plot(dfdaymin, stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "DEV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "standby_statistics = dfdaymin.T.describe(percentiles=[0.1,0.5,0.9]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    charts.plot(standby_statistics[['10%', '50%', '90%']], stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get detailed profiles for the last day\n",
    "now = pd.Timestamp('now', tz='UTC')\n",
    "start_of_day = now - pd.Timedelta(hours=now.hour, minutes=now.minute, seconds=now.second)\n",
    "sensors = map(hp.find_sensor, dfdaymin.columns)\n",
    "df_details = hp.get_data(sensors = sensors, head=start_of_day)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boxplot approach.  Possible for a period of maximum +/- 2 weeks. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# choose a period\n",
    "look_back_days = 10\n",
    "start = now - pd.Timedelta(days=look_back_days)\n",
    "dfdaymin_period = dfdaymin.ix[start:].dropna(axis=1, how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "box = [dfdaymin_period.loc[i,:].dropna().values for i in dfdaymin_period.index]\n",
    "for sensor in dfdaymin_period.columns:\n",
    "    plt.figure(figsize=(10,5))\n",
    "    ax1=plt.subplot(121)\n",
    "    ax1.boxplot(box, positions=range(len(box)), notch=False)\n",
    "    ax1.plot(range(len(box)), dfdaymin_period[sensor], 'rD', ms=10, label='Sluipverbruik')\n",
    "    xticks = [x.strftime(format='%d/%m') for x in dfdaymin_period.index]\n",
    "    plt.xticks(range(len(box)), xticks, rotation='vertical')\n",
    "    plt.title(hp.find_sensor(sensor).device.key + ' - ' + sensor)\n",
    "    ax1.grid()\n",
    "    ax1.set_ylabel('Watt')\n",
    "    plt.legend(numpoints=1, frameon=False)\n",
    "    ax2=plt.subplot(122)\n",
    "    try:\n",
    "        ax2.plot_date(df_details[sensor].index, df_details[sensor].values, 'b-', label='Afgelopen nacht')\n",
    "        #ax2.xaxis_date() #Put timeseries plot in local time\n",
    "        # rotate the labels\n",
    "        plt.xticks(rotation='vertical')\n",
    "        ax2.set_ylabel('Watt')\n",
    "        ax2.grid()\n",
    "        plt.legend(loc='upper right', frameon=False)\n",
    "        plt.tight_layout()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    else:\n",
    "        plt.savefig(os.path.join(c.get('data', 'folder'), 'figures', 'standby_horizontal_'+sensor+'.png'), dpi=100)\n",
    "        pass\n",
    "    \n",
    "    if not DEV:\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Percentile approach.  Useful for longer time periods, but tweaking of graph still needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# choose a period\n",
    "look_back_days = 40\n",
    "start = now - pd.Timedelta(days=look_back_days)\n",
    "dfdaymin_period = dfdaymin.ix[start:].dropna(axis=1, how='all')\n",
    "df = dfdaymin_period.join(standby_statistics[['10%', '50%', '90%']], how='left')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for sensor in dfdaymin_period.columns:\n",
    "    plt.figure(figsize=(10,8))\n",
    "    ax1=plt.subplot(211)\n",
    "    ax1.plot_date(df.index, df[u'10%'], '-', lw=2, color='g', label=u'10% percentile')\n",
    "    ax1.plot_date(df.index, df[u'50%'], '-', lw=2, color='orange', label=u'50% percentile')\n",
    "    ax1.plot_date(df.index, df[u'90%'], '-', lw=2, color='r', label=u'90% percentile')\n",
    "    ax1.plot_date(df.index, df[sensor], 'rD', ms=7, label='Your standby power') \n",
    "    ax1.legend()\n",
    "    locs, lables=plt.xticks()\n",
    "    xticks = [x.strftime(format='%d/%m') for x in num2date(locs)]\n",
    "    plt.xticks(locs, xticks, rotation='vertical')\n",
    "    plt.title(hp.find_sensor(sensor).device.key + ' - ' + sensor)\n",
    "    ax1.grid()\n",
    "    ax1.set_ylabel('Watt')\n",
    "    \n",
    "    ax2=plt.subplot(212)\n",
    "    try:\n",
    "        ax2.plot_date(df_details[sensor].index, df_details[sensor].values, 'b-', label='Afgelopen nacht')\n",
    "        #ax2.xaxis_date() #Put timeseries plot in local time\n",
    "        # rotate the labels\n",
    "        plt.xticks(rotation='vertical')\n",
    "        ax2.set_ylabel('Watt')\n",
    "        ax2.grid()\n",
    "        plt.legend(loc='upper right', frameon=False)\n",
    "        plt.tight_layout()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    else:\n",
    "        plt.savefig(os.path.join(c.get('data', 'folder'), 'figures', 'standby_vertical_'+sensor+'.png'), dpi=100)\n",
    "        pass\n",
    "    if not DEV:\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
