__author__ = 'thorwhalen'

import numpy as np
from ut.datapath import datapath
import ut.daf.diagnosis as daf_diagnosis
import ut.parse.google as google
import ut.parse.util as parse_util
import pandas as pd
import ut.daf.ch as daf_ch

import os
import re
from ut.pstr.trans import toascii
from . import reporting
from datetime import datetime
from ut.util.ulist import ascertain_list
from ut.util.ulist import all_true
from ut.pstr.trans import to_unicode_or_bust
from serialize.data_accessor import DataAccessor

split_exp = re.compile("[^&\w]*")

travel_domain_list = ['expedia', 'tripadvisor', 'trivago', 'marriott', 'booking', 'hotels', 'lastminute', 'accorhotels',
                      'kayak', 'venere', 'hilton', 'hotelscombined', 'agoda', 'choicehotels', 'travelocity',
                      'travelsupermarket', 'bestwestern', 'laterooms', 'radissonblu', 'hotwire', 'lonelyplanet',
                      'orbitz', 'starwoodhotels', 'frommers', 'hotel', 'hotelclub', 'hrs', 'novotel', 'wego', 'wotif',
                      'hoteltravel', 'hyatt', 'ibis', 'ihg', 'mercure', 'priceline', 'qualityinn',
                      'beoo', 'easytobook', 'ebookers', 'hostelbookers', 'lq', 'melia', 'millenniumhotels', 'mrandmrssmith',
                      'nh-hotels', 'ratestogo', 'sofitel', 'tablethotels', 'travelandleisure']

html_data = DataAccessor('html/google_results_tests')

def get_search_term_html(search_term):
    file_name = html_data+'.html'
    try:
        return html_data.loads(file_name)
    except:
        IOError("didn't find %s" % html_data.filepath(file_name))

def google_light_parse(gresult):
    gresult = parse_util.x_to_soup(gresult)
    parse_dict = dict()
    resultStats = input.find(name='div',attrs={'id':'resultStats'})
    if resultStats:
        parse_dict['_resultStats'] = google.parse_number_of_results(resultStats)




def save_search_term_that_does_not_have_num_of_results(search_term):
    print("no num_of_results in: %s" % search_term)

def add_travel_score(query_report_df, html_folder=datapath(),ext='.html'):
    pass


# TODO: Continue coding add_travel_score()
def mk_search_term_domains_df(query_report_df, html_folder=datapath(),ext='.html'):
    search_terms = np.unique(list(query_report_df['search_term']))
    domain_lists = []
    search_term_list = []
    for st in search_terms:
        filename = os.path.join(html_folder,st+ext)
        print(filename)
        if os.path.exists(filename):
            search_term_list.append(st)
            domain_lists.append(get_domain_list_from_google_results(filename))
    return pd.DataFrame({'search_term':search_term_list, 'domain_list':domain_lists})

def add_search_term_ndups(df, count_var='ndups'):
    d = df[['search_term']].groupby('search_term').count()
    d.columns = [count_var]
    return df.merge(d, left_on='search_term', right_index=True)

def add_target_scores(query_report_df):
    vars_to_keep = ['search_term','impressions','destination','ad_group','destination_imps_freq_fanout_ratio','ad_group_imps_freq_fanout_ratio']
    query_report_df = add_query_fanout_scores(query_report_df)
    query_report_df = query_report_df[vars_to_keep]
    query_report_df = daf_ch.ch_col_names(query_report_df,
                                   ['ad_group_score','destination_score'],
                                   ['ad_group_imps_freq_fanout_ratio','destination_imps_freq_fanout_ratio'])
    query_report_df = query_report_df.sort(columns=['search_term','destination_score','ad_group_score'])
    return query_report_df

def add_query_fanout_scores(query_report_df):
    if 'destination' not in query_report_df.columns:
        query_report_df = add_destination(query_report_df)
    ad_group_fanout = mk_query_fanout_scores(query_report_df,target='ad_group',statVars='impressions',keep_statVars=False)
    ad_group_fanout = daf_ch.ch_col_names(ad_group_fanout,
                                   ['ad_group_imps_freq_fanout_ratio','ad_group_count_fanout_ratio'],
                                   ['impressions_freq_fanout_ratio','impressions_count_fanout_ratio'])
    destination_fanout = mk_query_fanout_scores(query_report_df,target='destination',statVars='impressions',keep_statVars=False)
    destination_fanout = daf_ch.ch_col_names(destination_fanout,
                                   ['destination_imps_freq_fanout_ratio','destination_count_fanout_ratio'],
                                   ['impressions_freq_fanout_ratio','impressions_count_fanout_ratio'])
    query_report_df = query_report_df.merge(ad_group_fanout,on=['search_term','ad_group'])
    query_report_df = query_report_df.merge(destination_fanout,on=['search_term','destination'])
    return query_report_df


def mk_query_fanout_scores(query_report_df, target='ad_group', statVars='impressions',keep_statVars=False):
    # target = ascertain_list(target)
    # if ('destination' in target) and ('destination' not in query_report_df.columns):
    #     query_report_df['destination'] = query_report_df['ad_group'].apply(lambda x : re.match('[^|]*',x).group(0))
    # if not all_true([x in query_report_df.columns for x in target]):
    #     raise ValueError("the dataframe doesn't have the column %s and I don't know how to make it" % target)
    if target not in query_report_df.columns:
        if target=='destination':
            query_report_df = add_destination(query_report_df)
        else:
            raise ValueError("the dataframe doesn't have the column %s and I don't know how to make it" % target)
    return daf_diagnosis.mk_fanout_score_df(query_report_df,fromVars=['search_term'],toVars=target,statVars=statVars,keep_statVars=keep_statVars)

def add_destination(query_report_df):
    assert 'ad_group' in query_report_df.columns, "you must have the variable ad_group to infer destination"
    # get the destination as the substring before the first |
    query_report_df['destination'] = query_report_df['destination'] = \
        query_report_df['ad_group'].apply(lambda x : re.match('[^|]*',x).group(0))
    return query_report_df

def get_domain_list_from_google_results(gresults):
    domain_list = []
    if not isinstance(gresults,dict): # assume it's a soup, html, or filename thereof
        gresults = google.parse_tag_dict(google.mk_gresult_tag_dict(gresults))
    # if not, assume the input is a info_dict
    if 'organic_results_list' in gresults:
        domain_list = domain_list + [x['domain'] for x in gresults['organic_results_list'] if 'domain' in x]
    if 'top_ads_list' in gresults:
        domain_list = domain_list + [x['disp_url_domain'] for x in gresults['top_ads_list'] if 'disp_url_domain' in x]
    if 'organic_results_list' in gresults:
        domain_list = domain_list + [x['disp_url_domain'] for x in gresults['organic_results_list'] if 'disp_url_domain' in x]
    return domain_list


def similarity_with_travel_domains(domain_list):
    if isinstance(domain_list, pd.DataFrame):
        domain_list = list(domain_list.index)
    return len(set(domain_list).intersection(set(travel_domain_list)))\
               / float(len(domain_list))



###############################################################################################
## SEMANTIC STUFF


def mk_term_count_from_google_results(gresults):
    """
    takes a google result (in the form of html, filename thereof, soup, or info_dict, and
    returns a Series whose indices are terms and values are term counts
    (the number of times the term appeared in the google result)
    """
    # get preprocessed text from gresults
    gresults = process_text_for_word_count(mk_text_from_google_results(gresults))
    # tokenize this text
    toks = tokenize_text(gresults)
    # make a dataframe of term counts TODO: Explore faster ways to do this
    df = pd.DataFrame(toks,columns=['token'])
    df = df.groupby('token').count()
    df.columns = ['count']
    df = df.sort(columns=['count'],ascending=False) # TODO: Take out sorting at some point since it's unecessary (just for diagnosis purposes)
    return df

def tokenize_text(gresult_text):
    return re.split(split_exp,gresult_text)

def process_text_for_word_count(text):
    """
    Preprocesses the text before it will be fed to the tokenizer.
    Here, we should put things like lower-casing the text, casting letters to "simple" ("ascii", "non-accentuated")
    letters, replacing some common strings (such as "bed and breakfast", "New York" by singular token representatives
    such as "b&b", "new_york"), and what ever needs to be done before tokens are retrieved from text.
    """
    return toascii(to_unicode_or_bust(text)).lower()


def mk_text_from_google_results(gresults):
    if not isinstance(gresults,dict): # if not a dict assume it's a soup, html, or filename thereof
        gresults = google.parse_tag_dict(google.mk_gresult_tag_dict(gresults))
    if 'organic_results_list' in gresults:
        title_text_concatinated = ' '.join([x['title_text'] for x in gresults['organic_results_list'] if 'title_text' in x])
        snippet_text_concatinated = ' '.join([x['st_text'] for x in gresults['organic_results_list'] if 'st_text' in x])
        text_concatinated = title_text_concatinated + ' ' + snippet_text_concatinated
    else:
        search_for_tag = ['_ires','_search','_res','_center_col']
        for t in search_for_tag:
            if t in gresults:
                text_concatinated = soup_to_text(gresults[t])
                break
        if not text_concatinated: # if you still don't have anything
            text_concatinated = soup_to_text(gresults) #... just get the text from the whole soup
    return text_concatinated

def soup_to_text(element):
    return list(filter(visible, element.findAll(text=True)))

def visible(element):
    if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
    elif re.match('<!--.*-->', str(element)):
        return False
    return True



###############################################################################################
## MAKING DATA

def mk_df_of_travel_domains():
    # set up resources
    html_folder = '/D/Dropbox/dev/py/data/html/google_results_tests/'
    file_list = ['hotel - 100 Google Search Results.html',
             'find hotel deals - 100 Google Search Results.html',
             'hotel travel sites - 100 Google Search Results.html',
             'find hotels - 100 Google Search Results.html',
             'hotel paris - 100 Google Search Results.html',
             'hotel rome - 100 Google Search Results.html',
             'hotel london - 100 Google Search Results.html',
             'hotel nyc - 100 Google Search Results.html',
             'hotels in france - 100 Google Search Results.html',
             'hotels in italy - 100 Google Search Results.html'
            ]
    filepath_list = [os.path.join(html_folder,f) for f in file_list]
    # parse all this
    r = [google.mk_gresult_tag_dict(f) for f in filepath_list]
    r = [google.parse_tag_dict(f) for f in r]
    # make domain lists
    org_domain_list = []
    ads_domain_list = []
    tads_domain_list = []
    for rr in r:
        rrr = rr['organic_results_list']
        org_domain_list = org_domain_list + [x['domain'] for x in rrr if 'domain' in x]
        rrr = rr['rhs_ads_list']
        ads_domain_list = ads_domain_list + [x['disp_url_domain'] for x in rrr if 'disp_url_domain' in x]
        rrr = rr['top_ads_list']
        ads_domain_list = ads_domain_list + [x['disp_url_domain'] for x in rrr if 'disp_url_domain' in x]
    domain_list = org_domain_list + ads_domain_list
    print("number of org_domain_list entries = %d" % len(org_domain_list))
    print("number of ads_domain_list entries = %d" % len(ads_domain_list))
    print("number of (all) domain_list entries = %d" % len(domain_list))
    # make a dataframe counting the number of times we encouter each domain
    df = pd.DataFrame(domain_list,columns=['domain'])
    dg = df.groupby('domain').count() #agg([('domain_count','len')])
    dg = daf_ch.ch_col_names(dg,'count','domain')
    thresh = 4
    print("length before removing count<%d entries = %d" % (thresh,len(dg)))
    dg = dg[dg['count']>=thresh]
    print("length before removing count<%d entries = %d" % (thresh,len(dg)))
    dg['frequency'] = dg['count']/float(max(dg['count']))
    dg = dg.sort(columns=['count'],ascending=False)
    dg.head(30)
    # return this!
    return dg


