# -*- coding: utf-8 -*-
"""Demo.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/github/tresoldi/catcoocc/blob/master/docs/Demo.ipynb

# Categorical Co-occurrences

`catcoocc` is a library for facilitating the analysis of categorical co-occurrence by means of symmetric matrices, asymmetric matrices, and association rules. It is intended for usage on three main types of data: collections of pairwise sequences (both aligned and not), categorical fields in databases, and matrices of presence/absence (such as for investigation of species co-occurrence in biology). More than statistical significance, interested in effect size and strength of association.

For the purposes of the library, the following definitions are used:

- a co-occurrence is an observation of two categorical elements from two different series
- a population is a collection of co-occurrences which can come from the product of elements observed in two sequences (or, if the sequences are aligned, by n-gram windows), by the properties of an individual in a population, or by the product of presences
- a scorer is a collection of association measures between all the possible products of values in alphabet_x and alphabet_y (including those not observed in the data)
- scorers can be computed according to different methods (such as MLE...), either on raw counts of observed data or on smoothed methods; likewise, the output of scorers can be scaled in different ways
- scorers can be symmetric (such as those coming from chi2 methods) or assymetric (such a Theil U), and can be either restricted to positive values or include negative values, and can be bound or not to a range
"""

# This cell contains the system commands for locally installing the package;
# please remember that in most cases, especially in the playground,
# Google CoLab requires the runtime to be restarted (CTRL+M) in order for it
# to pick the new package (thus: run this cell, restart the runtime, and
# run other cells)
#!rm -rf catcoocc
#!git clone https://github.com/tresoldi/catcoocc.git
#!pip3 install --force-reinstall --no-deps -e catcoocc/

#from IPython.display import HTML, display

# Import 3rd party libraries
from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
import tabulate

# Import own libraries
import catcoocc
from catcoocc.scorer import CatScorer

def plot_scorer(scorer, alpha_x, alpha_y, title=None, font_scale=2, figsize=(25, 25)):
    if not title:
        title = ""

    matrix = pd.DataFrame(scorer, index=alpha_y, columns=alpha_x)

    sns.set(font_scale=font_scale)
    plt.figure(figsize=figsize)

    ax = plt.subplot(111)
    sns.heatmap(matrix, annot=True, fmt='.2f', linewidths=.5, center=0, ax=ax).set_title(title, fontsize=font_scale*figsize[0])

"""Let's start exploring using a subset of the mushroom dataset.

| class     | cap-shape |
|-----------|-----------|
| poisonous | convex    |
| edible    | convex    |
| edible    | bell      |
| poisonous | convex    |
| edible    | convex    |
| edible    | convex    |
| edible    | bell      |
| edible    | bell      |
| poisonous | convex    |
| edible    | bell      |
| edible    | convex    |
| edible    | convex    |
| edible    | bell      |
| poisonous | convex    |
| edible    | convex    |
| edible    | sunken    |
| edible    | flat      |
| poisonous | convex    |
| poisonous | convex    |
| poisonous | convex    |

Mushrooms can be either edible or poisonous, and the cap shapre can be of four types, convex, bell, flat, or sunken. In this subset, flat and sunken are underrepresented, with a single case, both being edibles, but of course it is not safe to eat it?
"""

mushroom_data = catcoocc.read_sequences("docs/mushroom-small.tsv")
mushroom_cooccs = catcoocc.collect_cooccs(mushroom_data)
scorer = catcoocc.scorer.CatScorer(mushroom_cooccs)

mle = scorer.mle()
pmi = scorer.pmi()
npmi = scorer.pmi(True)
chi2 = scorer.chi2()
chi2_ns = scorer.chi2(False)
cramersv = scorer.cramers_v()
cramersv_ns = scorer.cramers_v(False)
fisher = scorer.fisher()
theil_u = scorer.theil_u()
catcoocc_i = scorer.catcoocc_i()
catcoocc_ii = scorer.catcoocc_ii()

headers = [
    'pair',
    'mle_0',        'mle_1',
    'pmi_0',        'pmi_1',
    'npmi_0',       'npmi_1',
    'chi2_0',       'chi2_1',
    'chi2ns_0',     'chi2ns_1',
    'cremersv_0',   'cremersv_1',
    'cremersvns_0', 'cremersvns_1',
    'fisher_0',     'fisher_1',
    'theilu_0',     'theilu_1',
    'catcoocci_0',  'catcoocci_1',
    'catcooccii_0', 'catcooccii_1',
]

table = []
for pair in sorted(scorer.obs):
    buf = [
        pair,
        "%0.4f" % mle[pair][0],         "%0.4f" % mle[pair][1],
        "%0.4f" % pmi[pair][0],         "%0.4f" % pmi[pair][1],
        "%0.4f" % npmi[pair][0],        "%0.4f" % npmi[pair][1],
        "%0.4f" % chi2[pair][0],        "%0.4f" % chi2[pair][1],
        "%0.4f" % chi2_ns[pair][0],     "%0.4f" % chi2_ns[pair][1],
        "%0.4f" % cramersv[pair][0],    "%0.4f" % cramersv[pair][1],
        "%0.4f" % cramersv_ns[pair][0], "%0.4f" % cramersv_ns[pair][1],
        "%0.4f" % fisher[pair][0],      "%0.4f" % fisher[pair][1],
        "%0.4f" % theil_u[pair][0],     "%0.4f" % theil_u[pair][1],
        "%0.4f" % catcoocc_i[pair][0],  "%0.4f" % catcoocc_i[pair][1],
        "%0.4f" % catcoocc_ii[pair][0], "%0.4f" % catcoocc_ii[pair][1],
    ]
    table.append(buf)

    
print(tabulate.tabulate(table, headers=headers, tablefmt='markdown'))

"""and now more"""

xy, yx, alpha_x, alpha_y = catcoocc.scorer.scorer2matrices(catcoocc_ii)
plot_scorer(xy, alpha_x, alpha_y, "x->y", 2, (15, 15))

"""and even more"""

plot_scorer(yx, alpha_y, alpha_x, "y->x", 2, (15, 15))
