#!/usr/bin/env python
# -*- coding: UTF-8 -*-

# Script for importing Barnet Council's data. 
# Assumes all level 2 keys have level 1 parents. 

import os, sys, csv
import datapkg
from pylons import config

import wdmmg.model as model
import util
from wdmmg.lib import loader

barnet_string = unicode('Barnet')
barnet_council_string = unicode('Barnet Council')
dataset_name = unicode('barnet')

# NB gah, should have autogenerated these! rather than copying the COFOGMapper.
# Unlike COFOG these codes have no external significance. Fix in future.
temp_dict = [
     { "cat": "Schools", "code": "01", 
     "lower_levels": {"Primary Schools" : u"01.1",
    "Secondary Schools" : u"01.2",
    "Centrally Retained Schools Budget" : u"01.3",
    "Special Schools" : u"01.4",
    "Pre School" : u"01.5"} }, 
     { "cat": "Keeping Children Safe", "code": "02", 
     "lower_levels": {"Children in Care" : u"02.1",
    "Services for Schools" : u"02.2",
    "Support for Vulnerable Families" : u"02.3",
    "Disabled Children’s Services" : u"02.4",
    "Safeguarding" : u"02.5", 
    "Fostering and Adoption  - Internal" : u"02.6", 
    "Youth Activities" : u"02.7",} },
     { "cat": "Housing Benefits", "code": "03", 
     "lower_levels": {} },
     { "cat": "Environment and operations", "code": "04", 
     "lower_levels": {
	    "Street Cleaning" : u"04.1", 
	    "Transport" : u"04.2", 
	    "Waste Collection (plus NLWA)" : u"04.3", 
	    "Highways and Pavement Maintenance" : u"04.4", 
	    "Street Lighting" : u"04.5", 
	    "Greenspaces" : u"04.6", 
	    "Parking" : u"04.7", 
	    "Trading Standards and Community Safety (Community Protection Group)" : u"04.8",
      } },
     { "cat": "Adult Social Services", "code": "05", 
     "lower_levels": {
	    "Registered Care Homes" : u"05.1", 
	    "Support in the Home" : u"05.2",  
	    "Day Services" : u"05.3",  
	    "Social Work and Safeguarding" : u"05.4",  
	    "Direct Payments" : u"05.5", 
	    "Prevention - Keeping people Independent and Carer Support" : u"05.6",
      } },
     { "cat": "Corporate Services", "code": "06", 
     "lower_levels": {
	    "Central Expenses" : u"06.1",
	    "IT" : u"06.2",
	    "Tax Collection and Finance" : u"06.3",
	    "HR" : u"06.4",
	    "Finance" : u"06.5",
		"Management and Business Support" : u"06.6",
	    "Customer Services" : u"06.7",
		"Customer  Services" : u"06.7",
      } },
     { "cat": "Chief Executive's Service", "code": "07", 
     "lower_levels": {
      } },
     { "cat": "Corporate Governance", "code": "08", 
     "lower_levels": {
	    "Democratic Services and Elections" : u"08.1",
	    "Legal Services" : u"08.2",
	    "Internal Assurance" : u"08.3",
	    "Management and Business Support" : u"08.4",
	    "Tax collection and Finance" : u"08.5",
      } },
     { "cat": "Planning, Housing and Regeneration", "code": "09", 
     "lower_levels": {
	    "Planning" : u"09.1", 
	    "Assisting Homeless People" : u"09.2", 
	    "Environmental Health" : u"09.3", 
	    "Leisure" : u"09.4", 
	    "Procurement and Grants" : u"09.5",
      } },
     { "cat": "Commercial Services", "code": "10", 
     "lower_levels": {
	    "Management and Business Support" : u"10.1", 
		"Management and Business Suppor" : u"10.1", 
	    "Property, Public Office and Facilities" : u"10.2",
      } },
     { "cat": "Libraries", "code": "11", 
     "lower_levels": {
      } },
]

def load_file(filename, commit_every=None):
    '''
    Loads a file from `filename` into a dataset with name 'barnet'.
    The file should be CSV formatted.
    
    filename - a filename

    '''
    # Semaphore to prevent the data being loaded twice.
    assert not (model.Session.query(model.Dataset)
        .filter_by(name=dataset_name)
        ).first(), 'Barnet already loaded'
            # Make a new dataset.
    dataset_ = model.Dataset(name=dataset_name, currency=u'gbp', notes=u'''
Barnet Council spending data.

Source data package: custom data supplied by Barnet Council
        ''')
    # Retrieve or create the required Keys.
    key_from = util.get_or_create_key(name=u'from', notes=u'''\
The entity that the money was paid from.''')
    key_to = util.get_or_create_key(name=u'to', notes=u'''\
The entity that the money was paid to.''')
    key_time = util.get_or_create_key(name=u'time', notes=u'''\
The accounting period in which the spending happened.''')
    key_barnet_level_1 = util.get_or_create_key(name=u'key_barnet_level_1', notes=u'''\
Barnet Council Level 1 spending.''')
    key_barnet_level_2 = util.get_or_create_key(name=u'key_barnet_level_2', notes=u'''\
Barnet Council Level 2 spending.''')

    # Make sure the 'parent' key exists
    key_parent = model.Session.query(model.Key).filter_by(name=u'parent').first()
    if not key_parent:
        key_parent = model.Key(name=u'parent', notes=u'Means "is part of".')
        model.Session.add(key_parent)
    key_barnet_level_2.keyvalues[key_parent] = key_barnet_level_1.name
    model.Session.commit()

    # Utility function for creating EnumerationValues.
    # Works in a special way for Barnet, because we take codes from our dictionary.
    def get_or_create_value(key, name, code=None, notes=u'', index={}, parent=None):
        print 'get_or_create_value(key, name, code)', key, name, code
        if not code:
            temp_name = name.encode("utf-8")
            if parent: 
                for d in temp_dict:
                    if d['cat'] == parent:
                        lower_levels = d['lower_levels']
                        code = lower_levels[temp_name]
            else: 
                for d in temp_dict:
                    if d['cat'] == temp_name:
                        code = d['code']
        print "get_or_create_value: " + str(code)
        if (key.name, code) not in index:
            if parent:
                ev = model.EnumerationValue(key=key, code=code, name=name or code, notes=notes)
                ev.keyvalues[key_parent] = parent
                index[key.name, code] = ev
                model.Session.add(ev)
            else:
                index[key.name, code] = model.EnumerationValue(
                    key=key, code=code, name=name or code, notes=notes)
                model.Session.add(index[key.name, code])
        return index[key.name, code]

    barnet = get_or_create_value(key_from, name=barnet_string, code=barnet_string, notes=barnet_council_string)
    barnet_code = barnet.code

    # The 'To' key is always Society, in this instance
    society = get_or_create_value(
        key=key_to, code=u'society',
        name=u'Society (the General Public)',
        notes= u'''A dummy entity to be the recipient of final government
spending'''
        )
    model.Session.commit()
    society_id = society.code

    # Utility function for parsing numbers.
    def to_float(s):
        if not s: return 0.0
        return float(s.replace(',', ''))

    # Read in the CSV file: create a list of items - 
    # these hold just the actual entrys (no aggregates)
    items = []
    reader = csv.reader(open(filename, "rU"))
    header = reader.next()
    for row_index, row in enumerate(reader):
        # Parse row: strip and convert to unicode
        row = [unicode(x.decode("mac_roman").strip()) for x in row]
        level_1 = row[0]
        level_2 = row[1]
        amount = row[3]     
        description = row[4]
        #print description
        print "Our row: ", row
        items = {}
        items[key_from] = barnet_code
        items[key_to] = society_id
        items[key_time] = '2009-2010'
        # Make the Entry and its ClassificationItems.
        txn = model.Entry(dataset_=dataset_, amount=amount)
        for key, code in items.items():
            model.Session.add(model.ClassificationItem(
                entry=txn,
                value=get_or_create_value(key, code, code)
            ))
        # add level 1 and 2 classification
        if level_2 == "":
              print "adding level 1 item"
              model.Session.add(model.ClassificationItem(
                   entry=txn,
                   value=get_or_create_value(key_barnet_level_1, level_1, notes=description)
              )) 
              model.Session.commit()       
        else: 
              print "adding level 2 item"   
              model.Session.add(model.ClassificationItem(
                   entry=txn,
                   value=get_or_create_value(key_barnet_level_1, level_1, notes='Click on smaller bubbles for more detail')
              ))
              model.Session.add(model.ClassificationItem(
                   entry=txn,
                   value=get_or_create_value(key_barnet_level_2, level_2, parent=level_1, notes=description)  
              ))   
              model.Session.commit()        
        model.Session.commit()


def drop():
    '''
    Drops from the database all records associated with dataset 'barnet'. 
    '''
    # Delete only the keys we created ourselves.
    # TODO: Move as many as possible of these into separate data packages.
    raise NotImplemented

def load():
    '''
    Downloads the Barnet data, and loads it into the database with dataset name 'barnet'.
    '''
    # Get the CRA data package.
    pkgspec = 'file://%s' % os.path.join(config['getdata_cache'], 'barnet')
    # Load the data: second argument is a list of keys used to classify spending
    load_file( os.path.join(config['getdata_cache'], 'barnet') + '/barnet_clean.csv', commit_every=1000)
    # Commit everything before exiting
    model.Session.commit()
    model.Session.remove()

