import os
import time
import json
import webbrowser
import subprocess, sys
from datetime import datetime, timezone, timedelta
from dateutil.relativedelta import relativedelta
import copy

import sqlite3 
from contextlib import closing
from pathlib import Path

import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk, GLib, Gdk
gi.require_version('Notify', '0.7')
from gi.repository import Notify as notify

import conf
notify.init(conf.app_name)

lists = {}

db_file = conf.user_data_dir+"/data.db"

# debug_level = 0 # production value 
debug_level = 2 # dev value 
debug_systems = []



def dbg(*data, **kwargs):
    ''' Any number of positional args then kwargs 
    l=level {0:Error, 1:Warning, 2:Info, 3:Details}  
    s=system (Debuggable systems: 'taskwindow','signals','todoloading','user_settings', 'targets')'''

    levels = {0:'Error', 1:'Warning', 2:'Info', 3:'Details'}

    # default system and level
    system = ''
    level = 2

    if 's' in kwargs: system = kwargs['s']

    if 'l' in kwargs: level = int(kwargs['l'])

    if "all" in debug_systems or system in debug_systems or level <= debug_level: 

        if 'notify' in kwargs:
            # Will this cause a segfault if called from a bg thread?
            notify.Notification.new(conf.app_name+" "+levels[level], str(kwargs['notify']), None).show()

        # o = "Debug "+system+" "+levels[level], tuple(filter(None,data))
        o = system+" "+levels[level], tuple(filter(None,data))
        pretty_print(o)

    # else:
    #     # debug the debug
    #     print("Not displaying dbg ")
    #     # pretty_print(data)


def error_notice(title, details = None, e = None):
    print('ERROR',title,details,e)
    notify.Notification.new(str(title),str(details), None).show()


def pretty_dict(i, item_sep = ", ", use_indents=True, indent=''):
    o = ''
    if use_indents:
        o += indent

    if isinstance(i,dict):
        for key, val in i.items():

            o += str(key).capitalize() + ": "
            if isinstance(val,(dict,list,tuple)): 
                o += "\n" + pretty_dict(val, item_sep, use_indents, indent + '    ')
            else: 
                o += str(val) + item_sep

    elif isinstance(i,(list,tuple)):
        for val in i:
            if isinstance(val,(dict,list,tuple)): 
                o += "\n" + pretty_dict(val, item_sep, use_indents, indent + '    ')
            else: 
                o += str(val) + item_sep
    else:
        o += str(i) + item_sep

    return o


def pretty_print(i):
    print(pretty_dict(i))


def time_to_sec(time=None):
    if not time:
        return 0
    sec = 0
    for k, v in enumerate(reversed(time.split(':'))):
        sec += int(v) * 60 ** k
    return sec

def sec_to_time(sec):

    try:
        int(sec)
    except Exception as e:
        print("sec to time error", e)
        sec = 0

    time = str("{:02d}".format(int(sec // 3600))) + ':' + str("{:02d}".format(int((sec % 3600) // 60))) + ':' + str("{:02d}".format(int(sec % 60)))
    return time.lstrip('0').lstrip(':')

def now():
    return datetime.now()
    # return datetime.now(timezone.utc)
    # return datetime.now(timezone.utc).astimezone().tzinfo

def start_of_day():
    return now().replace(hour=0,minute=0,second=0,microsecond=0)

def time_difference(earlier, later = None):
    ''' Return seconds (float), later defaults to now'''
    # date_format = "%Y-%m-%dT%H:%M:%S.%f%z" # This can probably be updated / tossed (ActivityWatch format )
    if type(earlier) == str:
        earlier = datetime.strptime(earlier,'%Y-%m-%dT %H:%M:%S')

    if not later:
        later = now() 

    difference = later - earlier

    return difference.total_seconds()


def get_connector_openable(widget = None, connector_dict = None, open_it = True):

    for option in ['file','front_end_url','url','uri','open_command']:

        if option in connector_dict:
            if open_it:
                open_external(connector_dict[option])
                return True
            else:
                return connector_dict[option]

    print('ERROR: get_connector_openable failed to find an openable')
    print(connector_dict)
    return False

def open_external(i):
    # if platform.system() == "Windows":
    #     os.startfile(folder_path)
    # else:
    if(i.startswith('http')):
        webbrowser.open(i,1,True)
    else:
        opener = "open" if sys.platform == "darwin" else "xdg-open"
        subprocess.call([opener, i])



def get_lists_for_item(i): 
    ''' i can be a task or a list
    Returns {'ids':['l_1','l_12','l_123],'labels':['todolist label','project','sub project']}'''

    lists = lists_cache()
    # pretty_print(lists)

    ids = []
    labels = []

    if i and 'parent_id' in i and i['parent_id'] and i['parent_id'] in lists:

        l = lists[i['parent_id']]

        while l:
            ids.insert(0, l['id'])
            labels.insert(0, l['label'])
            if l['parent_id'] in lists:
                l = lists[l['parent_id']]
            else: 
                break

    # This may need empty val filtering . see: https://stackoverflow.com/a/3845449/4692205

    output = {'ids':ids,'labels':labels}

    # print('get_lists_for_item',i['label'])
    # pretty_print(output)

    return output


def extended_label(i):
    if not i:
        dbg('emptiness passed to extended_label',i)
        return ''

    lists = get_lists_for_item(i)
    l = lists['labels']
    l_ids = lists['ids']

    if conf.user['display_todolist_as_top_level_list'] != 'always':
        if l_ids and l_ids[0] == i['todolist']:
            del l[0]

    l.append(i['label'])
    o = ' > '.join(l)
    return o


def lists_cache(new_lists = None):
    ''' This is useful but very gimpy. If new_lists is supplied it will exclusively use that (until it is replaced or the global lists var is emptied)'''
    global lists
    
    if(new_lists):
        lists = new_lists
        # TODO: add this to the existing rather than overwriting...
    
    elif(lists == {}):
        lists = db_query('SELECT * FROM lists WHERE status = 1 ORDER BY extended_label DESC',None, 'id') # TODO: limit to active todolists

        print('list_cache l',lists)
        for l in lists.items():
            lists[l['id']]['data'] = json.loads(l['data'])

    return lists 


def first(i, default = None):

    if isinstance(i, (tuple,list)):
        return i[0]

    elif isinstance(i,dict):
        return  next(iter(i.values()))


def save_user_settings():
    print("Save updated user_settings")
    dbg(conf.user)
    with open(conf.user_settings_dir+"/user_settings.json","w") as settings_file:
        json.dump(conf.user, settings_file)


def db_query(sql,parameters=None,key=None,error_handling=1):
    '''error_handling: 2 = raise error, 1 = notify error, 0 = ignore errors '''
    # print('db_query parameters type: ',type(parameters))
    # print('first(parameters)',first(parameters))
    global db_file   
    c = sqlite3.connect(db_file)
    c.row_factory = sqlite3.Row
    try:
        with c:
            if parameters:
                
                if isinstance(parameters, (dict,list)) and isinstance(first(parameters), (dict)):
                    if isinstance(parameters, dict):
                        parameters = list(parameters.values())
                    
                    rows = c.executemany(sql,parameters).fetchall()
                        
                else: 
                    rows = c.execute(sql,parameters).fetchall()

            else:
                rows = c.execute(sql).fetchall()

            result = [dict(row) for row in rows]

            if key and result and key in result[0]:
                o = {}
                for row in result:
                    o[row[key]] = row
                result = o
            # print("db_query result",result)
        c.close()
        return result

    except Exception as e:
        dbg(e,"sql",sql,'parameters',parameters,s="db",l=0)
        if error_handling > 0:
            error_notice("database error",str(e))
            if error_handling > 1:
                raise e
        if key: 
            return {}
        else:
            return []


def db_init():
    if not os.path.isfile(db_file):
        print('initializing database')

        db_query("CREATE TABLE lists (id TEXT, label TEXT DEFAULT '', parent_id TEXT DEFAULT '', parent_label TEXT DEFAULT '', todolist TEXT DEFAULT '', priority INTEGER DEFAULT 0, status INTEGER DEFAULT 1, extended_label TEXT DEFAULT '', data TEXT DEFAULT '{}');")

        db_query("CREATE TABLE tasks (id TEXT, label TEXT DEFAULT '', parent_id TEXT DEFAULT '', parent_label TEXT  DEFAULT '', todolist TEXT DEFAULT '', priority INTEGER DEFAULT 0, status INTEGER DEFAULT 1, extended_label TEXT, data TEXT DEFAULT '{}');")
        
        db_query("CREATE TABLE sessions (start_time TEXT, duration INTEGER, task_id TEXT, parent_id TEXT, todolist TEXT, extended_label TEXT, notes TEXT, timetracker TEXT);")

        db_query("CREATE TABLE system (field TEXT PRIMARY KEY NOT NULL, value TEXT);")

        db_query("INSERT INTO system(field, value) VALUES('db_schema_version', '0.4')")
        

def db_update():

    try:
        db_schema_version = float(db_query("SELECT field, value FROM system WHERE field = 'db_schema_version' ",error_handling=2)[0]['value'])
    except Exception as e:
        print("Updating db_schema_version to 0.2")

        db_query("CREATE TABLE system (field TEXT PRIMARY KEY NOT NULL, value TEXT)")
        db_query("REPLACE INTO system(field, value) VALUES('db_schema_version', '0.2')")

        db_schema_version = float(db_query("SELECT field, value FROM system WHERE field = 'db_schema_version' ")[0]['value'])

        db_query("ALTER TABLE lists ADD COLUMN status INTEGER DEFAULT 1")
        db_query("ALTER TABLE tasks ADD COLUMN status INTEGER DEFAULT 1")

        db_query("ALTER TABLE lists ADD COLUMN extended_label TEXT")
        db_query("ALTER TABLE tasks ADD COLUMN extended_label TEXT")

        db_query("ALTER TABLE sessions ADD COLUMN extended_label TEXT")

        # add default values for priority and data 
        
        # Since these column are just a cache, replace instead of copying columns
        db_query("ALTER TABLE tasks DROP COLUMN priority")
        db_query("ALTER TABLE tasks DROP COLUMN data")
        db_query("ALTER TABLE lists DROP COLUMN priority")
        db_query("ALTER TABLE lists DROP COLUMN data")

        # add new columns
        db_query("ALTER TABLE lists ADD COLUMN priority INTEGER DEFAULT 0")
        db_query("ALTER TABLE lists ADD COLUMN data TEXT DEFAULT '{}'")
        db_query("ALTER TABLE tasks ADD COLUMN priority INTEGER DEFAULT 0")
        db_query("ALTER TABLE tasks ADD COLUMN data TEXT DEFAULT '{}'")


    if db_schema_version == 0.2:
                
        db_query("ALTER TABLE lists DROP COLUMN status")
        db_query("ALTER TABLE tasks DROP COLUMN status")

        db_query("ALTER TABLE lists ADD COLUMN status INTEGER DEFAULT 1")
        db_query("ALTER TABLE tasks ADD COLUMN status INTEGER DEFAULT 1")

        db_query("REPLACE INTO system(field, value) VALUES('db_schema_version', '0.3')")
    
        db_schema_version = 0.3


    if db_schema_version == 0.3:
        db_query("ALTER TABLE sessions ADD COLUMN timetracker TEXT")
        db_query("ALTER TABLE sessions ADD COLUMN notes TEXT")
        db_query("REPLACE INTO system(field, value) VALUES('db_schema_version', '0.4')")
        print('adding timetracker column to session table')

        # for s in db_query("SELECT start_time, task_id, todolist FROM sessions"):

        for todolist_id, todo in conf.user['todolists'].items():
            print("set timetracker to ",todo['timetracker'], " for sessions from ",todolist_id)

            db_query("UPDATE sessions SET timetracker = ? WHERE todolist = ?",(todo['timetracker'],todolist_id) )

        db_schema_version = 0.4

    # if db_schema_version == 0.4:


    dbg('db_schema_version', db_schema_version,s='db')

def db_to_py_type_map(db_type, value=None):
    types = {
        'INTEGER':int,
        'BOOLEAN':bool,
        'REAL':float,
        'TEXT':str,
        'BLOB':bytes,
    }

    if isinstance(value, (dict,list,tuple)) and db_type in ['TEXT','BLOB']:
        return json.dumps(value)

    if db_type in types: 
        if value:
            return types[db_type](value)
        else:
            types[db_type]


def db_columns(table, cache = {}):
    ''' Returns the PRAGMA query, from cache '''

    if table not in cache: 

        if table not in ['lists','tasks','sessions','system']:
            raise ValueError("bad table passed to db_columns!")
        else:
            fields = db_query("PRAGMA table_info("+table+")")

            # sadly "Parameter markers can be used only for expressions, i.e., values. You cannot use them for identifiers like table and column names." 
            # fields = db_query("PRAGMA table_info(?)",(table,)) 
            
            cache[table] = fields

    return cache[table]


def time_target_priority(i, table = "tasks"):

    if 'priority' in i and i['priority'] > 0:
        # pass through existing priority
        # NOTE: Maybe de-prioritize here if it has surpassed a max target?
        return i['priority']
    else:
        time_target_ballance = check_time_target(i, table, 'min')
        if time_target_ballance and time_target_ballance > 0:
            # This could have more ... nuance...
            i['priority'] = 1 

        else: 
            i['priority'] = 0

    return i['priority'] 


def db_prepare_item(i,table = "tasks"):
    if table not in ['lists','tasks']:
        raise ValueError("bad table passed to db_prepare!")

    fields = db_columns(table)

    o = {}

    if not 'status' in i:
        i['status'] = 1

    if not 'extended_label' in i:
        i['extended_label'] = extended_label(i)

    # time_target_ballance = check_time_target(i, table, 'min')
    # if time_target_ballance and time_target_ballance > 0:
    #     dbg("prioritize min time_target task in db_prepare")
    #     i['priority'] = 1

    # print('fields',fields)
    for f in fields:
        if f['name'] in i:
            o[f['name']] = db_to_py_type_map(f['type'],i[f['name']])
        else:
            o[f['name']] = f['dflt_value']
    
    return o


def db_set_item(i,table = "tasks"):
    i = db_prepare_item(i,table)

    columns = ', '.join(i.keys())
    placeholders = ':'+', :'.join(i.keys())

    db_query('INSERT INTO '+table+' (%s) VALUES (%s)' % (columns, placeholders),[i])


def db_set_todolist(todolist_id,lists,tasks):
    # print('db_set_todolist params: \ntodolist_id: ',todolist_id,'lists: ',lists,'tasks: ',tasks)

    insert_tasks = {}
    insert_lists = {}

    # use the parameter-provided lists for extended label instead of the existing (db) lists
    lists_cache(lists)
        
    for id, i in lists.items():
        insert_lists[id] = db_prepare_item(i,'lists')
    
    for id, i in tasks.items():
        insert_tasks[id] = db_prepare_item(i,'tasks')

    # Clear incomplete lists cache
    lists_cache({})

    # for testing
    # insert_tasks = {'3':insert_tasks['3'],'294':insert_tasks['294']}
    # print('insert_tasks: ',json.dumps(insert_tasks,indent=4))

    db_query("DELETE FROM tasks WHERE todolist = ?",(todolist_id,)) # What about removed 
    db_query("DELETE FROM lists WHERE todolist = ?",(todolist_id,))

    if insert_tasks:
        task_columns = insert_tasks[list(insert_tasks.keys())[0]].keys()
        columns = ', '.join(task_columns)
        placeholders = ':'+', :'.join(task_columns)

        db_query('INSERT INTO tasks (%s) VALUES (%s)' % (columns, placeholders),insert_tasks)
        
    if insert_lists:
        list_columns = insert_lists[list(insert_lists.keys())[0]].keys()
        columns = ', '.join(list_columns)
        placeholders = ':'+', :'.join(list_columns)

        db_query('INSERT INTO lists (%s) VALUES (%s)' % (columns, placeholders),insert_lists)


def db_cleanup():
    
    # ids = tuple(conf.user['todolists'].keys())
    ids = []
    # print('ids_sql',ids_sql)
    for id, todo in conf.user['todolists'].items():
        if todo['status']:
            ids.append(id)

    # Old save
    print("Active todolist ids",ids)

    # Delete unlinked tasks
    db_query('DELETE FROM tasks WHERE todolist NOT IN (%s)' % ','.join('?'*len(ids)),ids)
    # print(len(orphaned_tasks),"orphaned_tasks",orphaned_tasks)
    # orphaned_tasks = db_query('SELECT id, extended_label, todolist FROM tasks WHERE todolist NOT IN (%s)' % ','.join('?'*len(ids)),ids)
    # print(len(orphaned_tasks),"orphaned_tasks",orphaned_tasks)
    
    # Delete unlinked lists
    db_query('DELETE FROM lists WHERE todolist NOT IN (%s)' % ','.join('?'*len(ids)),ids)

    # orphaned_lists = db_query('SELECT id, extended_label, todolist FROM lists WHERE todolist NOT IN (%s)' % ','.join('?'*len(ids)),ids)
    # print("orphaned_lists",orphaned_lists)

    missing_task_sessions = db_query("SELECT * FROM sessions WHERE task_id NOT IN (SELECT id FROM tasks)")
    # print("missing_task_sessions",missing_task_sessions)
    print(len(missing_task_sessions))

    for s in missing_task_sessions:
        # print("missing_task_session")
        # print(pretty_dict(s))
        if s['extended_label']:
            possible_matches = db_query("SELECT * FROM tasks WHERE extended_label = ? ",(s['extended_label'],))
            if possible_matches:
                print("possible_matches")
                print(pretty_dict(possible_matches))

    # update extended_labels
    mismatched_sessions = db_query("SELECT * FROM sessions WHERE extended_label NOT IN (SELECT extended_label FROM tasks)")

    # db_query("UPDATE sessions SET extended_label = (SELECT extended_label FROM tasks WHERE id = task_id AND parent_id = parent_id) WHERE task_id IN (SELECT id FROM tasks) AND extended_label NOT IN (SELECT extended_label FROM tasks)")



    # SELECT * FROM sessions WHERE extended_label IS NULL;
    # UPDATE sessions SET extended_label = (SELECT extended_label FROM tasks WHERE id = task_id) WHERE extended_label IS NULL

    # db_query("SELECT label, todolist FROM tasks WHERE todolist NOT IN ?",(ids_sql,))
    # db_query("DELETE FROM tasks WHERE todolist  NOT IN "+ids_sql)
    # db_query("DELETE FROM lists WHERE todolist NOT IN "+ids_sql)

    # exit()

def db_get_item_by_id(id,table = 'tasks'):
    
    if table not in ['tasks','lists']:
        dbg('bad table "'+str(table)+'" passed to db_get_item_by_id',id,l=0)
        return {}

    try:
        data = db_query("SELECT * FROM "+table+" WHERE id = ?",(id,))
        # print(data)
        if data:
            return proc_db_item(data[0],table)
        else:
            dbg('db_get_item_by_id from '+table+' failed for id +',id,l=0)
            return {}

    except Exception as e:
        dbg('db_get_item_by_id failed',e,l=0)


def proc_db_item(i,table='tasks'):
    ''' Parse task or list data json and apply time target priority '''
    i['data'] = json.loads(i['data'])
    i['parent_label'] = str(i['parent_label']) #
    i['priority'] = time_target_priority(i,table)
    return i


def db_get_todolist(todolist_id):
    tasks = {}

    for t in db_query("SELECT * FROM tasks WHERE status = 1 AND todolist = ?",(todolist_id,)):
        tasks[t['id']] = proc_db_item(t)

    lists = {}
    for l in db_query("SELECT * FROM lists WHERE todolist = ? AND status = 1",(todolist_id,)):
        lists[l['id']] = proc_db_item(l,table='lists')

    o = {'lists': lists, 'tasks':tasks}
    return o


def db_save_session(session):
    prepared_session = {
        'start_time' : str(session['start_time'].strftime("%Y-%m-%d %H:%M:%S")),
        'duration': round(session['duration']),
        'task_id': str(session['task']['id']),
        'parent_id': str(session['task']['parent_id']),
        'todolist': str(session['task']['todolist']),
        'extended_label': str(session['extended_label']),
        'timetracker': str(session['timetracker']),
        'notes': str(session['notes']),
    } 
    db_query("INSERT INTO sessions(start_time, duration, task_id, parent_id, todolist, extended_label,timetracker, notes) VALUES(:start_time, :duration, :task_id, :parent_id, :todolist, :extended_label, :timetracker, :notes )",prepared_session)


def get_total_time(id, category = 'tasks', start_time = None, end_time = None, get_minutes = None):
    '''If start_time is int or float: it is treated as a number of days,
    Default returns num seconds, (unless get_minutes is True)
    end_time only work if start time is specified '''

    if category in ['list','lists']:
        where = 'parent_id'
    else:
        where = 'task_id'

    if start_time:
        
        if type(start_time) in [int,float]:
            start_time = start_of_day() - timedelta(days = start_time)
         
        if not end_time: end_time = now()
        
        result = db_query("SELECT sum(duration) as duration FROM sessions WHERE "+where+" = ? AND start_time > ? AND start_time < ? ",(id, start_time.strftime("%Y-%m-%d %H:%M:%S"), end_time.strftime("%Y-%m-%d %H:%M:%S")))
    else:
        result = db_query("SELECT sum(duration) as duration FROM sessions WHERE "+where+" = ? ",(id,))

    # TODO make the list version (reverse) recursive
    # print("get_total_time result",result)

    result = force_number(result[0]['duration'])

    if get_minutes:
        result = result and result / 60 or 0
    
    return result


def get_recent_tasks(count = 15):
    data = db_query("SELECT DISTINCT tasks.* FROM tasks JOIN sessions ON sessions.task_id = tasks.id WHERE tasks.status = 1 GROUP BY tasks.id ORDER BY MAX(sessions.start_time) DESC LIMIT ?",(count,))
    o = {}
    for t in data:
        o[t['id']] = proc_db_item(t)
    return o


def get_priority_tasks(count = 100):

    p_tasks = db_query("SELECT DISTINCT tasks.* FROM tasks WHERE priority > 0 AND status = 1 ORDER BY priority ASC LIMIT ?",(count,))

    tt_task_ids = list(conf.user['time_targets']['tasks'].keys())
    tt_list_ids = list(conf.user['time_targets']['lists'].keys())

    # TODO: [Performance] combine these two queries if possible
    tt_tasks = db_query('SELECT DISTINCT * FROM tasks WHERE id IN(%s)' % ','.join('?'*len(tt_task_ids)),tt_task_ids)
    
    tt_list_tasks = db_query('SELECT DISTINCT * FROM tasks WHERE parent_id IN(%s)' % ','.join('?'*len(tt_list_ids)),tt_list_ids)

    o = {}

    for t in p_tasks + tt_tasks + tt_list_tasks: 
    # for t in tt_tasks + tt_list_tasks: #for testing
        i = proc_db_item(t)
        if i['priority'] > 0: 
            o[t['id']] = t
        # else: 
            # dbg('de-prioritize '+ t['label'], i['priority'], s='targets')


    dbg('priority tasks', o, s='targets', l=3)
    return o


def force_number(i):
      try:
        o = float(i)
      except:
        o = 0
      return o


def get_times(task):
    day = start_of_day()
    week = day - timedelta(days = 7)
    month = day - timedelta(days = 30)

    o = {}
    task['parent_label'] = str(task['parent_label']) # This keeps causing a noneType + str concat error even though it should be a string already 

    o[task['label'] +' today'] = get_total_time(task['id'],'task',day)
    o[task['parent_label']+' today'] = get_total_time(task['parent_id'],'list',day)

    o[task['label'] +' weekly'] = get_total_time(task['id'],'task',week)
    o[task['parent_label']+' week'] = get_total_time(task['parent_id'],'list',week)    
    
    o[task['label'] +' monthly'] = get_total_time(task['id'],'task',month)
    o[task['parent_label']+' month'] = get_total_time(task['parent_id'],'list',month)

    for key, val in o.items():
        o[key] = sec_to_time(val)

    return o


def db_deactivate_todo(id):
    print("deactivating todolist",id)
    # Delete inactive todo items (except where there are sessions for them) 
    db_query("DELETE FROM tasks WHERE todolist = ? AND id NOT IN (SELECT task_id FROM sessions)",(id,))
    db_query("DELETE FROM lists WHERE todolist = ? AND id NOT IN (SELECT parent_id FROM tasks)",(id,))
    db_query("UPDATE lists SET status = -1 WHERE todolist = ?",(id,))
    db_query("UPDATE tasks SET status = -1 WHERE todolist = ?",(id,))


def get_todolists(use_db_cache = False):
    tasks = {}
    lists = {}  

    # TODO: Use threads to load todos concurrently?
    # with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
    #    executor.map(thread_function, range(3))
    
    for id, todo in conf.user['todolists'].items(): 
        if not todo['status']:
            db_deactivate_todo(id)
            continue # omit inactive connectors
        
        if use_db_cache:
            print('loading '+todo['type']+' todo '+id+" from db")

            todos = db_get_todolist(todo['id'])
        else:
            print('Refreshing '+todo['type']+' todo '+id+" ")

            try:
                todos = conf.todo_connectors[todo['type']].get_todos(todo)
                conf.todo_sync_time = now()
                db_set_todolist(todo['id'],todos['lists'],todos['tasks'])
            except Exception as e:
                # This will happen if offline

                # For testing
                if debug_level == 3 or 'todoloading' in debug_systems or todo['type'] in debug_systems :
                    raise e

                error_notice('Error Loading '+todo['label'],e)
                try:
                    todos = db_get_todolist(todo['id'])
                except Exception as e:
                    error_notice('Also Failed to load '+todo['label']+' From cache' ,e)
                    todos = {'lists': {}, 'tasks':{}}

        tasks.update(todos['tasks'])
        lists.update(todos['lists'])
    
    lists_cache(lists)

    o = {'lists': lists, 'tasks':tasks}
    return o

def get_most_recent_list(session = None):

    if session and 'parent_id' in session: 
       return  session['parent_id']

    last_session = db_query("SELECT parent_id FROM sessions WHERE parent_id IN (SELECT id FROM lists WHERE status = 1) ORDER BY sessions.start_time DESC LIMIT 1")
    if last_session: 
        return last_session[0]['parent_id']


def choose_from_lists(callback, selected_list_id = None, session = None, accepts_tasks = True):
    ''' Returns a Gtk.ScrolledWindow widget with radio button'''

    # get last used list
    if not selected_list_id:
        selected_list_id = get_most_recent_list(session = None)

    box = Gtk.VBox()

    scrolled_window = Gtk.ScrolledWindow()
    scrolled_window.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC)
    scrolled_window.set_propagate_natural_height(True)
    scrolled_window.set_shadow_type(Gtk.ShadowType.ETCHED_IN)
    scrolled_window.set_size_request(-1, 400)
    # scrolled_window.set_min_content_height(500)
    scrolled_window.add(box)

    todolist = None

    where = " WHERE status = '1' "

    if accepts_tasks:
        where += " AND data like '%\"accepts_tasks\": true%' "

    lists = db_query("SELECT * FROM lists "+where+" ORDER BY todolist, extended_label ASC ", None, 'id')

    for id, l in lists.items():
        l['data'] = json.loads(l['data']) 
        # print("AddTask list ",l['label'])

        if todolist == None:
            # initial button group
            button_group = Gtk.RadioButton(label=l['label'])
            item = button_group
        else:
            item = Gtk.RadioButton(label=extended_label(l), group=button_group)

        if todolist != l['todolist']:
            # new buttongroup for new todolist
            todolist = l['todolist']
            l['header'] = Gtk.Label()
            l['header'].set_markup("<b>"+conf.user['todolists'][todolist]["label"]+"</b>")
            box.add(l['header'])
            
        if l['id'] == selected_list_id:
            item.set_active(True)

        item.connect("toggled", callback, l)
        box.add(item)

    return scrolled_window


def add_todos_to_menu(target_menu = None, menu_tasks = None, list_menus = None, activate_callback = None, use_db_cache = True):

    todos = get_todolists(use_db_cache)

    lists = todos['lists']  
    tasks = todos['tasks']

    # print('Add tasks to menu:')
    # print('refreshed lists',json.dumps(lists, indent=4))
    # print(json.dumps(tasks, indent=4))

    list_menu_items = {}
    priority_tasks = {}

    if conf.user['display_todolist_as_top_level_list'] == 'always':
        toplevel_todos = True
    elif conf.user['display_todolist_as_top_level_list'] == 'never':
        toplevel_todos = False
    else:
        # Count top level lists and nest if to many 
        toplevel_count = 0

        # items = lists
        items = lists | tasks
        for id, i in items.items():
            if i['status'] == 1 and i['parent_id'] == i['todolist']:
                toplevel_count += 1
            elif 'priority' in i and i['priority']:
                # count priority tasks
                toplevel_count += 1


        print('num top level lists',toplevel_count)
        dbg('toplevel_count',toplevel_count,s='menu')


        if toplevel_count > conf.user['max_top_level_menu_items']:
            toplevel_todos = True
        else:
            toplevel_todos = False
    
    # for testing
    # toplevel_todos = True
    # toplevel_todos = False
    
    # separator after priority tasks
    target_menu.prepend(Gtk.SeparatorMenuItem.new())

    # Create list sub menus 
    for list_id, l in lists.items():
        if l['status'] != 1:
            continue

        if not toplevel_todos and l['id'] == l['todolist']:
            continue

        list_menus[list_id] = Gtk.Menu() # the sub_list that items get added to
        list_menu_items[list_id] = Gtk.MenuItem.new_with_label(lists[list_id]['label']) # the "item" that gets added 
        list_menu_items[list_id].set_submenu(list_menus[list_id])

    # Add sub menus (items) to parents (sub_menus)
    for list_id, l in lists.items():
        if l['status'] == 1:

            if not toplevel_todos and l['id'] == l['todolist']:
                continue
            
            if 'priority' in l and l['priority']:
                target_menu.prepend(list_menu_items[list_id])
            elif (toplevel_todos and l['id'] == l['todolist']) or (not toplevel_todos and l['parent_id'] == l['todolist']):
                target_menu.append(list_menu_items[list_id])
            else:
                list_menus[l['parent_id']].append(list_menu_items[list_id])



    for id, t in tasks.items():
        if t['status'] == 1:

            menu_tasks[t['id']] = Gtk.MenuItem.new_with_label(str(t['label']))
            menu_tasks[t['id']].connect("activate", activate_callback, t)


            if 'priority' in t and t['priority'] > 0 and ('priority' not in lists[t['parent_id']] or lists[t['parent_id']]['priority'] in [0,None]):
                # Hoist priority tasks, in correct order
                priority_tasks[t['id']] = t['priority']

            elif not t['parent_id'] or t['parent_id'] not in list_menus:
                target_menu.append(menu_tasks[t['id']])
            else:
                # print("add "+t['label']+" to sub menu "+t['parent_label'] )
                list_menus[t['parent_id']].append(menu_tasks[t['id']])


    # Sort and add priority tasks
    if priority_tasks:

        priority_tasks = dict(sorted(priority_tasks.items(), key=lambda item: item[1], reverse=True))
        for t_id in priority_tasks:
            target_menu.prepend(menu_tasks[t_id])

    
    # Ugly (but effectively?) remove empty lists
    for key, menuitem in list_menu_items.items():
        if len(menuitem.get_children()) == 0:
            print("remove empty list", lists[key]['label'])
            menuitem.destroy()


    # Do it again to remove upper level empties!
    for key, menuitem in list_menu_items.items():
        if len(menuitem.get_children()) == 0:
            print("remove empty list", lists[key]['label'])
            menuitem.destroy()

    target_menu.show_all()

def datetime_minus_calendar_unit(unit = 'days', num = 1, ref_date = None):
    ''' returns a datetime for the start of the unit num units ago (from the optional ref_date) 
    unit can be days, weeks, months, years 
    num  
    ''' 
    # Looks weird but...
    num = num - 1 
    if not ref_date:
        ref_date = start_of_day()
    if unit in ['day','days']:
        o = ref_date - timedelta(days=num) 
        if num > 0: o = o - timedelta(days=num)
    if unit in ['week','weeks']:
        o = ref_date - timedelta(days=ref_date.weekday())
        if num > 0: o = o + relativedelta(dt1= o, weeks= - num)
    if unit in ['month','months']:
        o = ref_date.replace(day=1)
        if num > 0: o = o + relativedelta(dt1= o, months= - num)
    if unit in ['year','years']:
        o = ref_date.replace(month=1,day=1)
        if num > 0: o = o + relativedelta(dt1= o, years= - num)
    return o

def get_time_target_start_time(t):
    ''' t must be a time target dict from conf.user'''



def check_time_target(i,item_type = 'tasks', min_max_type = None):

    ''' No target returns False 
     max or min returns float of target minus sessions ''' 

    # Backward compatibility 
    if item_type == 'task': item_type = 'tasks'
    if item_type == 'list': item_type = 'lists'

    ballance = False

    if i['id'] in conf.user['time_targets'][item_type]:
        t = conf.user['time_targets'][item_type][i['id']]
    elif i['parent_id'] in conf.user['time_targets']['lists']:
        # TODO: make this infinitely recursive using get lists_for_task
        t = conf.user['time_targets']['lists'][i['parent_id']]
    else:
        dbg(str(i['label']) + " check_time_target no applicable target:", s='targets', l=3 )
        return False
        
    if not min_max_type or t['type'] == min_max_type:
        target_start = datetime_minus_calendar_unit(t['within_unit'],t['within_value'])

        ballance = t['value'] - get_total_time(i['id'],item_type, target_start, None, "get_minutes")

        dbg(i['label'] + " check_time_target ballance:" + str(ballance), s='targets', l=2 )
        return ballance
        
    dbg(i['label'] + min_max_type + " check_time_target no applicable min_max_type target:", s='targets', l=3 )
    return False
