import datetime
import logging

import pandas as pd
import os

from glob import glob
from tqdm import tqdm

from wikipedia_tools.utils import properties, utils
from wikipedia_tools.processor import loader
from pathlib import Path


def get_wiki_popular_pages():
    wiki_popular_page_df = pd.read_csv(properties.WIKI_CLIMATE_CHANGE_POPULAR_PAGES)
    return wiki_popular_page_df[wiki_popular_page_df['Page title'].notna()]


def is_popular(title):
    popular_df = get_wiki_popular_pages()
    return len(popular_df[popular_df['Page title'] == title]) > 0


def is_important(title):
    popular_df = get_wiki_popular_pages()
    return popular_df[popular_df['Page title'] == title]['Importance'].values[
        0] if is_popular(title) else 'na'


def get_wiki_pages_overview(category="Climate_change"):
    """
    Processes all wikipages and generates 1 dataframe where each row represents 1 wikipage with info:
    page title, created on, last updated, initial content, latest content, edits numner, content edits count, unique users,
    is_popular (if it exists under Wikipedida Climate Change Project), is_important (if it is popular and flagged important by the wiki project)
    :return:
    """
    path = Path(properties.WIKI_OVERVIEW_CSV)
    if path.is_file():
        return pd.read_csv(properties.WIKI_OVERVIEW_CSV, )

    wiki_titles = loader.get_wikipedia_page_titles(categories=[category], corpus=properties.CORPUS)
    popular_pages_df = get_wiki_popular_pages()
    wiki_page_meta_arr = []

    print('there are {} wiki titles'.format(len(wiki_titles)))
    for title_code in tqdm(wiki_titles[category]):
        original_df = loader.get_wikipedia_page_data(title_code)

        if original_df is None:
            continue
        if len(original_df) == 0:
            print('{} has len 0'.format(title_code))
            continue
        _df = original_df.copy()

        wiki_page_meta = {}

        _df['timestamp_str'] = _df['timestamp']
        _df['timestamp'] = pd.to_datetime(_df['timestamp'], infer_datetime_format=True)
        _df = _df.sort_values(by="timestamp")

        init_content_df = _df[(~_df['content'].isna()) & (_df['content'] != '')].copy()
        init_content_df = init_content_df[init_content_df['timestamp'] == init_content_df['timestamp'].min()]
        if len(init_content_df) == 0:
            print('NO CONTENT: ', title_code)
            continue
        last_content = _df[_df['timestamp'] == _df['timestamp'].max()]['content'].values[0]
        edits_w_latest_content_df = (_df[(_df['content'] == last_content)])
        last_updated_df = edits_w_latest_content_df[
            edits_w_latest_content_df['timestamp'] == edits_w_latest_content_df['timestamp'].min()]

        wiki_page_meta['title_code'] = title_code
        wiki_page_meta['page_title'] = init_content_df['page'].values[0]
        wiki_page_meta['created_on'] = init_content_df['timestamp'].values[0]
        wiki_page_meta['init_content'] = init_content_df['content'].values[0]
        wiki_page_meta['last_updated'] = last_updated_df['timestamp'].values[0]
        wiki_page_meta['latest_content'] = last_updated_df['content'].values[0]
        wiki_page_meta['edits_number'] = (len(original_df['timestamp'].unique()))
        wiki_page_meta['content_edits_number'] = (
            len(_df.drop_duplicates(subset=['content'], keep='first', ignore_index=True)))
        wiki_page_meta['unique_users'] = len(original_df['user'].unique())

        wiki_page_meta['is_popular'] = is_popular(init_content_df['page'].values[0])
        wiki_page_meta['importance'] = popular_pages_df[popular_pages_df['Page title'] == title_code]['Importance'].values[
            0] if wiki_page_meta['is_popular'] else 'na'

        wiki_page_meta_arr.append(wiki_page_meta)

    meta_df = pd.DataFrame(wiki_page_meta_arr)
    meta_df.to_csv(properties.WIKI_OVERVIEW_CSV, index=False)
    return meta_df

#get_wiki_edits_num
def get_revision_count(only_popular=False, date_granularity=properties.DATE_MONTHLY, desc='MONTH-YEAR', category="Climate_change"):
    fname = properties.WIKI_EDITS_COUNT_CSV.format(('POPULAR' if only_popular else 'ALL'), desc)
    path = Path(fname)
    if path.is_file():
        return pd.read_csv(fname, index_col='abstracted_date')

    wiki_titles = loader.get_wikipedia_page_titles(categories=[category], corpus=properties.CORPUS)

    print('there are {} wiki titles'.format(len(wiki_titles)))

    result_df = pd.DataFrame()

    for title_code in tqdm(wiki_titles[category]):
        original_df = loader.get_wikipedia_page_data(title_code)
        title = original_df["page"].values[0]
        if (only_popular and not is_popular(title)) or (original_df is None) or len(original_df) == 0:
            continue

        _df = original_df.copy()

        _df['timestamp_str'] = _df['timestamp']
        _df['timestamp'] = pd.to_datetime(_df['timestamp'], infer_datetime_format=True)
        _df['abstracted_date'] = _df['timestamp'].dt.strftime(date_granularity)

        edits_count_df = _df['abstracted_date'].value_counts().to_frame(name='edits_count')
        edits_count_df.index.name = 'abstracted_date'
        result_df = pd.concat([result_df, edits_count_df]).groupby(['abstracted_date']).sum()

    result_df.reset_index().to_csv(fname, index=False)
    return result_df


#

def batch_revisions_per_period(period=properties.DATE_MONTHLY, desc='monthly', category="Climate_change", corpus=None):
    folder_root = properties.FOLDER_WIKI_BATCHES.format(desc)
    corpus=properties.CORPUS if  corpus  is None else corpus
    wiki_titles = loader.get_wikipedia_page_titles(categories=[category], corpus=corpus)

    print('there are {} wiki titles'.format(len(wiki_titles)))
    periords_str_lst = []
    for title_code in tqdm(wiki_titles[category]):
        original_df = loader.get_wikipedia_page_data(title_code)
        if (original_df is None) or len(original_df) == 0:
            continue
        _df = original_df.copy()

        _df['timestamp_str'] = _df['timestamp']
        _df['timestamp'] = pd.to_datetime(_df['timestamp'], infer_datetime_format=True)
        _df['abstracted_date'] = _df['timestamp'].dt.strftime(period)
        periords_str_lst.extend(_df['abstracted_date'].unique().tolist())
        for period_val, period_df in _df.groupby('abstracted_date'):
            destination = f"{folder_root}/{period_val}/{title_code}.parquet"
            utils.create_folder(f"{folder_root}/{period_val}")
            period_df.to_parquet(destination, index=False)
    periods_str_lst = list(set(periords_str_lst))
    periods_str_lst = sorted([datetime.strptime(x, period) for x in  periods_str_lst])

    return folder_root,periods_str_lst

# get_wiki_edits_per_period
def get_wikipedia_page_periodic_overview(only_popular=False, period=properties.DATE_YEAR, desc='YEAR', category="Climate_change"):
    fname = properties.WIKI_EDITS_PERIODIC_CSV.format(('POPULAR' if only_popular else 'ALL'), desc)
    path = Path(fname)
    if path.is_file():
        return pd.read_csv(fname)

    wiki_titles = loader.get_wikipedia_page_titles(categories=[category], corpus=properties.CORPUS)

    print('there are {} wiki titles'.format(len(wiki_titles)))

    result_arr = []
    for title_code in tqdm(wiki_titles[category]):
        original_df = loader.get_wikipedia_page_data(title_code)
        if (only_popular and not is_popular(original_df["page"].values[0])) or (original_df is None) or len(original_df) == 0:
            continue
        _df = original_df.copy()

        _df['timestamp_str'] = _df['timestamp']
        _df['timestamp'] = pd.to_datetime(_df['timestamp'], infer_datetime_format=True)
        _df['abstracted_date'] = _df['timestamp'].dt.strftime(period)

        for period_val, period_df in _df.groupby('abstracted_date'):
            latest_df = period_df[period_df['timestamp'] == period_df['timestamp'].max()]
            revision_per_period = {}
            revision_per_period['content'] = latest_df['content'].values[0]
            revision_per_period['title'] = latest_df['page'].values[0]
            revision_per_period['title_code'] = title_code
            revision_per_period['period'] = period_val
            revision_per_period['user_count'] = len(period_df['user'].unique())
            revision_per_period['users'] = period_df['user'].values.tolist()
            revision_per_period['revision_count'] = len(period_df)
            revision_per_period['timestamp'] = latest_df['timestamp'].values[0]
            revision_per_period['links'] = latest_df['links'].values[0]
            revision_per_period['categories'] = latest_df['categories'].values[0]
            revision_per_period['urls'] = latest_df['urls'].values[0]
            result_arr.append(revision_per_period)

    result_df = pd.DataFrame(result_arr)  # , index = ['period', 'title'])

    result_df.to_csv(fname, index=False)
    return result_df

