#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 29 13:15:15 2017

@author: m75380

Deals with inserting a table in Elasticsearch
"""

import json
import logging
import os
import time

from elasticsearch import Elasticsearch, client, RequestError
import pandas as pd

from .helpers import gen_index_settings, _gen_index_settings_from_analyzers

def pre_process_tab(tab):
    ''' Clean tab before insertion.'''
    for x in tab.columns:
        if tab.loc[:, x].dtype.kind == 'O':
            tab.loc[:, x] = tab[x].str.strip()
    return tab


def create_index(es, table_name, columns_to_index, default_analyzer='keyword', 
                 analyzer_definitions=None, analyzer_index_settings=None, 
                 force=False):
    '''Create a new empty Elasticsearch index (used to host documents).
    
    Parameters
    ----------
    es: instance of `Elasticsearch`
        Connection to Elasticsearch.
    table_name: str
        Name of the index in Elasticsearch.
    columns_to_index: dict 
        Keys are the columns to index and values are the Elasticsearch 
        analyzers to use for the corresponding column (in addition to the 
        default analyzer).
            
        Ex: {'col1': {'analyzerA', 'analyzerB'}, 
             'col2': {}, 
             'col3': 'analyzerB'}
    default_analyzer: str
        The Elasticsearch analyzer to use on all columns being indexed (use 
        `keyword` or similar).
    analyzer_definitions: dict or None
        Use analyzer_definitions to generate the index_settings. Input is a 
        `dict` with keys the analyzer name and values a dict with `tokenizer`, 
        `filter`, and `analyzer` fields. (Cannot use with analyzer_index_settings)
    analyzer_index_settings: dict or None
        Elasticsearch settings to define custom analyzers if any are used. 
        See Elasticsearch documentation on how to create custom analyzers.
        The object is the dict representation of the object passed on to 
        Elasticsearch during index creation. (Cannot use with 
        analyzer_definitions)
    force: bool
        whether or not to delete and re-create an index if the name 
        (`table_name`) is already associated to an existing index.
    '''
    
    ic = client.IndicesClient(es)
    
    if ic.exists(table_name) and force:
        ic.delete(table_name)
    
    assert (analyzer_index_settings is None) or (analyzer_definitions is None)
    
    # Generate index settings template for custom analyzers that are used
    # in columns to_index or as default analyzer
    if analyzer_definitions is not None:
        custom_analyzers = {y for x in columns_to_index.values() for y in x \
                            if y in analyzer_definitions}
        if default_analyzer in analyzer_definitions:
            custom_analyzers.add(default_analyzer)
            
        # Generate the ES settings for all custom analyzers
        analyzer_index_settings = _gen_index_settings_from_analyzers( \
                        [analyzer_definitions[x] for x in custom_analyzers])
    
    if not ic.exists(table_name):
        index_settings = gen_index_settings(default_analyzer, columns_to_index, analyzer_index_settings)
        try:
            ic.create(table_name, body=json.dumps(index_settings))  
        except RequestError as e:
            # If the error is caused by a missing config file, propose to 
            # create the config file 
            if e.info['error']['caused_by']['type'] == 'file_not_found_exception':
                new_message = e.__str__() + '\n\n(MERGE MACHINE)--> This may be due to ' \
                                'ES resource not being available. ' \
                                'Run es_gen_resource.py (in sudo) for this to work'
            import pdb; pdb.set_trace()
            raise Exception(new_message)

def update_analyzers(es, table_name, columns_to_index, default_analyzer='keyword', 
                 analyzer_index_settings=None, force=False):
    
    # Close
    
    # Put
    pass
    # Open
    

def index(es, ref_gen, table_name, testing=False, file_len=0, action='index'):
    '''Insert values from `ref_gen` in the Elasticsearch index.
    
    Parameters
    ----------
    es: instance of `Elasticsearch`
        Connection to Elasticsearch.
    ref_gen: generator of `pandas.DataFrame` objects
        Can be generated by `ref_gen=pd.read_csv(file_path, chunksize=XXX))`.
    table_name: str
        Name of the index in Elasticsearch to insert to.
    testing: bool
        Whether or not to refresh index at each insertion (for dev purposes).
    file_len: 
        Original file len to display estimated time (for convenience).
    action: str ("index" or "update")
        Whether to index new row or update
    '''
    
    ic = client.IndicesClient(es)
    
    # For efficiency, reset refresh interval
    # see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html
    if not testing:
        low_refresh = {"index" : {"refresh_interval" : "-1"}}
        ic.put_settings(low_refresh, table_name)
    
    # Bulk insert
    logging.info('Started indexing')    
    i = 0
    t_start = time.time()
    for ref_tab in ref_gen:
        ref_tab = pre_process_tab(ref_tab)
        body = ''
        for key, doc in ref_tab.where(ref_tab.notnull(), None).to_dict('index').items():
            #TODO: make function that limits bulk size
            index_order = json.dumps({
                                action: {
                                          "_index": table_name, 
                                          "_type": 'structure', 
                                          "_id": str(key)
                                         }
                                })
            body += index_order + '\n'
            if action == 'index':
                body += json.dumps(doc) + '\n'
            elif action == 'update':
                   body += json.dumps({"doc": doc}) + '\n'
        es.bulk(body)
        i += len(ref_tab)
        
        # Display progress
        t_cur = time.time()
        eta = (file_len - i) * (t_cur-t_start) / i
        logging.info('Indexed {0} rows / ETA: {1} s'.format(i, eta))

    # Back to default refresh
    if not testing:
        default_refresh = {"index" : {"refresh_interval" : "1s"}}
        ic.put_settings(default_refresh, table_name)        
        es.indices.refresh(index=table_name)



if __name__ == '__main__':
    
    columns_to_index = {
        'SIEGE': {},
        'SIRET': {},
        'SIREN': {},
        'NIC': {},
        'L1_NORMALISEE': {
            'french', 'whitespace', 'integers', 'n_grams', 'city'
        },
        'L4_NORMALISEE': {
            'french', 'whitespace', 'integers', 'n_grams'
        },
        'L6_NORMALISEE': {
            'french', 'whitespace', 'integers', 'n_grams'
        },
        'L1_DECLAREE': {
            'french', 'whitespace', 'integers', 'n_grams', 'city'
        },
        'L4_DECLAREE': {
            'french', 'whitespace', 'integers', 'n_grams'
        },
        'L6_DECLAREE': {
            'french', 'whitespace', 'integers', 'n_grams'
        },
        'LIBCOM': {
            'french', 'whitespace', 'n_grams', 'city'
        },
        'CEDEX': {},
        'ENSEIGNE': {
            'french', 'whitespace', 'integers', 'n_grams', 'city'
        },
        'NOMEN_LONG': {
            'french', 'whitespace', 'integers', 'n_grams', 'city'
        },
        #Keyword only 'LIBNATETAB': {},
        'LIBAPET': {},
        'PRODEN': {},
        'PRODET': {}
    }
        

    #==============================================================================
    # Index in Elasticsearch 
    #==============================================================================
    testing = False
    force = True
    do_indexing = True
    chunksize = 2000
    
    ref_file_name = 'sirene_filtered.csv' # 'petit_sirene.csv'
    ref_sep = ',' #';'
    ref_encoding = 'utf-8' #'windows-1252'
    
    if testing:
        nrows = 10000
    else:
        nrows = 10**10
    
    ref_gen = pd.read_csv(os.path.join('local_test_data', 'sirene', ref_file_name), 
                      sep=ref_sep, encoding=ref_encoding,
                      usecols=columns_to_index.keys(),
                      dtype=str, chunksize=chunksize, nrows=nrows) 
    
    
    if testing:
        table_name = '123vivalalgerie4'
    else:
        table_name = '123vivalalgerie2'
    
    es = Elasticsearch(timeout=60, max_retries=10, retry_on_timeout=True)
    
    # https://www.elastic.co/guide/en/elasticsearch/reference/1.4/analysis-edgengram-tokenizer.html

    create_index(es, table_name, columns_to_index, force)

    if do_indexing:
        index(ref_gen, table_name, testing)
