{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Server running in the folder /mnt/data/work/opengrid/code/opengrid/notebooks at 127.0.0.1:49601\n",
      "Environment configured for development: True\n"
     ]
    }
   ],
   "source": [
    "# opengrid imports\n",
    "from opengrid.library import misc, houseprint, caching\n",
    "from opengrid.library.analysis import DailyAgg\n",
    "from opengrid import config\n",
    "c=config.Config()\n",
    "\n",
    "# other imports\n",
    "import pandas as pd\n",
    "import charts\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime as dt\n",
    "import pytz\n",
    "BXL = pytz.timezone('Europe/Brussels')\n",
    "\n",
    "\n",
    "# configuration for the plots\n",
    "DEV = c.get('env', 'type') == 'dev' # DEV is True if we are in development environment, False if on the droplet\n",
    "print(\"Environment configured for development: {}\".format(DEV))\n",
    "if not DEV:\n",
    "    # production environment: don't try to display plots\n",
    "    import matplotlib\n",
    "    matplotlib.use('Agg')\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.dates import MinuteLocator, HourLocator, DateFormatter, AutoDateLocator, num2date\n",
    "\n",
    "if DEV:\n",
    "    if c.get('env', 'plots') == 'inline':\n",
    "        %matplotlib inline\n",
    "    else:\n",
    "        %matplotlib qt\n",
    "else:\n",
    "    pass # don't try to render plots\n",
    "plt.rcParams['figure.figsize'] = 12,8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opening connection to Houseprint sheet\n",
      "Opening spreadsheets\n",
      "Parsing spreadsheets\n",
      "23 Sites created\n",
      "23 Devices created\n",
      "73 sensors created\n",
      "Houseprint parsing complete\n",
      "Using tmpo database from /mnt/data/work/opengrid/data/.tmpo/tmpo.sqlite3\n"
     ]
    }
   ],
   "source": [
    "hp = houseprint.Houseprint()\n",
    "sensors = hp.get_sensors(sensortype='electricity') # sensor objects\n",
    "\n",
    "# Remove some sensors\n",
    "exclude = [\n",
    "            '565de0a7dc64d8370aa321491217b85f' # 3E\n",
    "          ]\n",
    "solar = [x.key for x in hp.search_sensors(type='electricity', system='solar')]\n",
    "exclude += solar\n",
    "\n",
    "for s in sensors:\n",
    "    if s.key in exclude:\n",
    "        sensors.remove(s)\n",
    "\n",
    "hp.init_tmpo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/requests/packages/urllib3/connection.py:264: SubjectAltNameWarning: Certificate for api.flukso.net has no `subjectAltName`, falling back to check for a `commonName` for now. This feature is being removed by major browsers and deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 for details.)\n",
      "  SubjectAltNameWarning\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n",
      "WARNING:requests.packages.urllib3.connectionpool:Connection pool is full, discarding connection: api.flukso.net\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "No JSON object could be decoded",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-da1439b0e71c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mhp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msync_tmpos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m/mnt/data/work/opengrid/code/opengrid/opengrid/library/houseprint/houseprint.pyc\u001b[0m in \u001b[0;36msync_tmpos\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    420\u001b[0m         \"\"\"\n\u001b[0;32m    421\u001b[0m         \u001b[0mtmpos\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_tmpos\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 422\u001b[1;33m         \u001b[0mtmpos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    423\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    424\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msensors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msensortype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhead\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtail\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdiff\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'default'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresample\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'min'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munit\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'default'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/mnt/data/work/opengrid/code/tmpo-py/tmpo/__init__.pyc\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    133\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdbcur\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSQL_TMPO_TABLE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    134\u001b[0m             \u001b[1;31m# execute function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 135\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    136\u001b[0m             \u001b[1;31m# commit everything and close connection\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    137\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdbcon\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcommit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/mnt/data/work/opengrid/code/tmpo-py/tmpo/__init__.pyc\u001b[0m in \u001b[0;36msync\u001b[1;34m(self, *sids)\u001b[0m\n\u001b[0;32m    197\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    198\u001b[0m                 \u001b[0mrid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlvl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbid\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 199\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_req_sync\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlvl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    200\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    201\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mdbcon\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/mnt/data/work/opengrid/code/tmpo-py/tmpo/__init__.pyc\u001b[0m in \u001b[0;36m_req_sync\u001b[1;34m(self, sid, rid, lvl, bid)\u001b[0m\n\u001b[0;32m    303\u001b[0m         \u001b[0mr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    304\u001b[0m         \u001b[0mfs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 305\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    306\u001b[0m             fs.append((t, self._req_block(\n\u001b[0;32m    307\u001b[0m                 sid, token, t[\"rid\"], t[\"lvl\"], t[\"bid\"], t[\"ext\"])))\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/requests/models.pyc\u001b[0m in \u001b[0;36mjson\u001b[1;34m(self, **kwargs)\u001b[0m\n\u001b[0;32m    803\u001b[0m                     \u001b[1;31m# used.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    804\u001b[0m                     \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 805\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mcomplexjson\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    806\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    807\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/json/__init__.pyc\u001b[0m in \u001b[0;36mloads\u001b[1;34m(s, encoding, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[0;32m    336\u001b[0m             \u001b[0mparse_int\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mparse_float\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    337\u001b[0m             parse_constant is None and object_pairs_hook is None and not kw):\n\u001b[1;32m--> 338\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_decoder\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    339\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mcls\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    340\u001b[0m         \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mJSONDecoder\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/json/decoder.pyc\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, s, _w)\u001b[0m\n\u001b[0;32m    364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    365\u001b[0m         \"\"\"\n\u001b[1;32m--> 366\u001b[1;33m         \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraw_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    367\u001b[0m         \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    368\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/json/decoder.pyc\u001b[0m in \u001b[0;36mraw_decode\u001b[1;34m(self, s, idx)\u001b[0m\n\u001b[0;32m    382\u001b[0m             \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscan_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0midx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    383\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 384\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"No JSON object could be decoded\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    385\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: No JSON object could be decoded"
     ]
    }
   ],
   "source": [
    "hp.sync_tmpos()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The first time, this will take a very looong time to get all the detailed data for building the cache\n",
    "# Afterwards, this is quick\n",
    "starttime = dt.time(0, tzinfo=BXL)\n",
    "endtime = dt.time(5, tzinfo=BXL)\n",
    "caching.cache_results(hp=hp, sensors=sensors, resultname='elec_min_night_0-5', AnalysisClass=DailyAgg,  \n",
    "                      agg='min', starttime=starttime, endtime=endtime)\n",
    "\n",
    "caching.cache_results(hp=hp, sensors=sensors, resultname='elec_max_night_0-5', AnalysisClass=DailyAgg, \n",
    "                      agg='max', starttime=starttime, endtime=endtime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cache_min = caching.Cache(variable='elec_min_night_0-5')\n",
    "cache_max = caching.Cache(variable='elec_max_night_0-5')\n",
    "dfdaymin = cache_min.get(sensors=sensors)\n",
    "dfdaymax = cache_max.get(sensors=sensors)\n",
    "dfdaymin.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The next plot shows that some periods are missing.  Due to the cumulative nature of the electricity counter, we still have the total consumption.  However, it is spread out of the entire period.  So we don't know the standby power during these days, and we have to remove those days.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    sensor = hp.search_sensors(key='3aa4')[0]\n",
    "    df = sensor.get_data(head=pd.Timestamp('20151117'), tail=pd.Timestamp('20160104'))\n",
    "    charts.plot(df, stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def is_submeter(sensor, dfdaymin, dfdaymax):\n",
    "    \"\"\"\n",
    "    Return True if this sensor is a sub-meter\n",
    "    \n",
    "    sensor = sensor object\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    other_sensors = sensor.device.get_sensors(sensortype='electricity')\n",
    "    other_sensors.remove(sensor)\n",
    "    if len(other_sensors) == 0:\n",
    "        print(\"\\n{} - {}: no other sensors, this must be main.\".format(sensor.device.key, sensor.description))\n",
    "        return False\n",
    "    else:\n",
    "        print(\"\\n{} - {}: comparing with:\".format(sensor.device.key, sensor.description))\n",
    "        for o in other_sensors:\n",
    "            # we only check the values for last day\n",
    "            print(\"* {}:\".format(o.description))\n",
    "            sensormin = dfdaymin.ix[-1,sensor.key]\n",
    "            sensormax = dfdaymax.ix[-1,sensor.key]\n",
    "            try:\n",
    "                othermin = dfdaymin.ix[-1].dropna()[o.key]\n",
    "                othermax = dfdaymax.ix[-1].dropna()[o.key]\n",
    "            except:\n",
    "                print(\"  No data found for sensor {}\".format(o.description))\n",
    "                pass\n",
    "            else:\n",
    "                if (sensormin <= othermin) and (sensormax <= othermax):\n",
    "                    print(\"  {} has lower daily min AND max, so it is a submeter.\".format(sensor.description))                   \n",
    "                    return True\n",
    "                else:\n",
    "                    print(\"  {} has higher daily min and/or max, we look further.\".format(sensor.description))\n",
    "        else:\n",
    "            print(\"All other sensors have no data OR lower daily min and max. {} must be main.\".format(sensor.description))\n",
    "            return False\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# The function is_submeter makes one obvious error: see results for FL03001566\n",
    "for col in dfdaymin:\n",
    "    is_submeter(hp.find_sensor(col), dfdaymin, dfdaymax)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Clean out the data: \n",
    "# First remove sensors that are submeters\n",
    "for col in dfdaymin:\n",
    "    if is_submeter(hp.find_sensor(col), dfdaymin, dfdaymax):\n",
    "        print(\"\\n!!Removing submeter {}\".format(col))\n",
    "        dfdaymin = dfdaymin.drop(col, axis=1)\n",
    "# Now remove days with too low values to be realistic\n",
    "dfdaymin[dfdaymin < 10] = np.nan\n",
    "# Now remove days where the minimum=maximum (within 1 Watt difference)\n",
    "dfdaymin[(dfdaymax - dfdaymin) < 1] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dfdaymin.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    charts.plot(dfdaymin, stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "standby_statistics = dfdaymin.T.describe(percentiles=[0.1,0.5,0.9]).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if DEV:\n",
    "    charts.plot(standby_statistics[['10%', '50%', '90%']], stock=True, show='inline')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get detailed profiles for the last day\n",
    "now = pd.Timestamp('now', tz=BXL)\n",
    "dt_start_of_last_day = pd.Timestamp(dfdaymin.index[-1].date(), tz=BXL)\n",
    "dt_end_of_last_day = dt_start_of_last_day + pd.Timedelta(hours=endtime.hour, minutes=endtime.minute)\n",
    "sensors = map(hp.find_sensor, dfdaymin.columns)\n",
    "df_details = hp.get_data(sensors = sensors, head=dt_start_of_last_day, tail=dt_end_of_last_day)\n",
    "df_details.fillna(method='ffill', inplace=True)\n",
    "df_details.fillna(method='bfill', inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Boxplot approach.  Possible for a period of maximum +/- 2 weeks. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# choose a period\n",
    "look_back_days = 10\n",
    "dt_start_of_period = dt_start_of_last_day - pd.Timedelta(days=look_back_days-1)\n",
    "dfdaymin_period = dfdaymin.ix[dt_start_of_period:].dropna(axis=1, how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "box = [dfdaymin_period.loc[i,:].dropna().values for i in dfdaymin_period.index]\n",
    "for sensor in dfdaymin_period.columns:\n",
    "    fig=plt.figure(figsize=(10,5))\n",
    "    ax1=plt.subplot(121)\n",
    "    ax1.boxplot(box, positions=range(len(box)), notch=False)\n",
    "    ax1.plot(range(len(box)), dfdaymin_period[sensor], 'rD', ms=10, label='Standby power')\n",
    "    xticks = [x.strftime(format='%d/%m') for x in dfdaymin_period.index]\n",
    "    plt.xticks(range(len(box)), xticks, rotation='vertical')\n",
    "    plt.title(hp.find_sensor(sensor).device.key + ' - ' + hp.find_sensor(sensor).description)\n",
    "    ax1.grid()\n",
    "    ax1.set_ylabel('Watt')\n",
    "    plt.legend(numpoints=1, frameon=False)\n",
    "    ax2=plt.subplot(122)\n",
    "    try:\n",
    "        ax2.plot_date(df_details[sensor].index, df_details[sensor].values, 'b-', label='Last night')\n",
    "        ax2.xaxis_date(tz=BXL) #Put timeseries plot in local time\n",
    "        # rotate the labels\n",
    "        plt.xticks(rotation='vertical')\n",
    "        ax2.set_ylabel('Watt')\n",
    "        ax2.set_xlabel('Local time (BXL)')\n",
    "        ax2.grid()\n",
    "        \n",
    "        xax = ax2.get_xaxis() # get the x-axis\n",
    "        xax.set_major_locator(HourLocator())\n",
    "        xax.set_minor_locator(MinuteLocator(30))\n",
    "        \n",
    "        adf = xax.get_major_formatter() # the the auto-formatter\n",
    "\n",
    "        adf.scaled[1./24] = '%H:%M'  # set the < 1d scale to H:M\n",
    "        adf.scaled[1.0] = '%Y-%m-%d' # set the > 1d < 1m scale to Y-m-d\n",
    "        adf.scaled[30.] = '%Y-%m' # set the > 1m < 1Y scale to Y-m\n",
    "        adf.scaled[365.] = '%Y' # set the > 1y scale to Y\n",
    "\n",
    "        plt.legend(loc='upper right', frameon=False)\n",
    "        plt.tight_layout()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    else:\n",
    "        plt.savefig(os.path.join(c.get('data', 'folder'), 'figures', 'standby_horizontal_'+sensor+'.png'), dpi=100)\n",
    "        pass\n",
    "    \n",
    "    if not DEV:\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Percentile approach.  Useful for longer time periods, but tweaking of graph still needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# choose a period\n",
    "look_back_days = 40\n",
    "dt_start_of_period = dt_start_of_last_day - pd.Timedelta(days=look_back_days-1)\n",
    "dfdaymin_period = dfdaymin.ix[dt_start_of_period:].dropna(axis=1, how='all')\n",
    "df = dfdaymin_period.join(standby_statistics[['10%', '50%', '90%']], how='left')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for sensor in dfdaymin_period.columns:\n",
    "    plt.figure(figsize=(10,8))\n",
    "    ax1=plt.subplot(211)\n",
    "    ax1.plot_date(df.index, df[u'10%'], '-', lw=2, color='g', label=u'10% percentile')\n",
    "    ax1.plot_date(df.index, df[u'50%'], '-', lw=2, color='orange', label=u'50% percentile')\n",
    "    ax1.plot_date(df.index, df[u'90%'], '-', lw=2, color='r', label=u'90% percentile')\n",
    "    ax1.plot_date(df.index, df[sensor], 'rD', ms=7, label='Your standby power') \n",
    "    ax1.legend()\n",
    "    locs, lables=plt.xticks()\n",
    "    xticks = [x.strftime(format='%d/%m') for x in num2date(locs)]\n",
    "    plt.xticks(locs, xticks, rotation='vertical')\n",
    "    plt.title(hp.find_sensor(sensor).device.key + ' - ' + sensor)\n",
    "    ax1.grid()\n",
    "    ax1.set_ylabel('Watt')\n",
    "    \n",
    "    ax2=plt.subplot(212)\n",
    "    try:\n",
    "        ax2.plot_date(df_details[sensor].index, df_details[sensor].values, 'b-', label='Detailed consumption of last night')\n",
    "        ax2.xaxis_date(tz=BXL) #Put timeseries plot in local time\n",
    "        # rotate the labels\n",
    "        plt.xticks(rotation='vertical')\n",
    "        ax2.set_ylabel('Watt')\n",
    "        ax2.set_xlabel('Local time (BXL)')\n",
    "        ax2.grid()\n",
    "        \n",
    "        xax = ax2.get_xaxis() # get the x-axis\n",
    "        xax.set_major_locator(HourLocator())\n",
    "        xax.set_minor_locator(MinuteLocator(30))\n",
    "        \n",
    "        adf = xax.get_major_formatter() # the the auto-formatter\n",
    "\n",
    "        adf.scaled[1./24] = '%H:%M'  # set the < 1d scale to H:M\n",
    "        adf.scaled[1.0] = '%Y-%m-%d' # set the > 1d < 1m scale to Y-m-d\n",
    "        adf.scaled[30.] = '%Y-%m' # set the > 1m < 1Y scale to Y-m\n",
    "        adf.scaled[365.] = '%Y' # set the > 1y scale to Y\n",
    "        \n",
    "        plt.legend(loc='upper right', frameon=False)\n",
    "        plt.tight_layout()\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    else:\n",
    "        plt.savefig(os.path.join(c.get('data', 'folder'), 'figures', 'standby_vertical_'+sensor+'.png'), dpi=100)\n",
    "        pass\n",
    "    if not DEV:\n",
    "        plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
