---
title: "model Comparison"
author: "K. Enevoldsen"
date: "3/24/2021"
output: html_document
---

```{r}
library(tidyverse)
library(kableExtra)
library(knitr)
library(jsonlite)
library(magick)

source("utility.R")
```

```{r extract metrics}

models = tribble(~"Framework", ~"Model",
                "DaCy small", "Ælæctra Cased",
                "DaCy medium", "DaBERT",
                "DaCy large", "XLM-Roberta",
                "DaConvBERT", "DaConvBERT",
                "DaELECTRA", "DaELECTRA"
                )

files = list.files("../../DaCy_training/metrics/", full.names = T)

performance = files %>% map_df(metrics_from_json) %>%
  mutate(Framework = recode(Framework, 
                            `dane_-l-ctra_cased` = "DaCy small",
                            dane_large = "DaCy large",
                            dane_medium = "DaCy medium",
                            `dane_-l-ctra_uncased` = "Ælæctra uncased",
                            dane_conv_small = "DaConvBERT",
                            dane_electra = "DaELECTRA")
         ) %>% 
  merge(models) %>% 
  select(Framework, Model, everything())
```


```{r previous results}
perf = tribble(
~"Framework", ~Model,           ~Accuracy, ~"Location", ~"Organization", ~"Person", ~"Avg F1" , ~UAS, ~LAS, ~WPS,
"DaNLP BERT",      "DaBERT",      NA,        83.90,      72.98,       92.82,       84.04,       NA,      NA, NA,
"Flair",     "BiLSTM",      97.97,     84.82,      62.95,       93.15,       81.78,       NA,      NA, NA,
"SpaCy (v2)", "Tok2Vec",    96.15,     75.96,      59.57,       87.87,       75.73,       81.36,   77.46, NA,
"Stanza",     "BiLSTMs",    97.75,     NA,         NA,          NA,          NA,          86.83,   84.19, NA,
"Polyglot",   "Word Embeddings" ,   NA,        64.95,      39.3,        78.74,       64.18,       NA,      NA, NA,
"NERDA",  "mBERT",NA,        80.75,      65.73,       92.66,       80.66,       NA,      NA, NA,
)
performance_comp = rbind(performance, perf)
```


```{r final}
non_final_models = c("Ælæctra uncased", "DaConvBERT", "DaELECTRA")

perf_final = performance_comp %>%
  filter(!Framework %in% non_final_models)
```


```{r make tables}

options(knitr.kable.NA = '')

make_table = function(data, high_columns=high){
  res = data %>% 
    kbl(., 
        format="latex", 
        booktabs = T, 
        align=c("l", "l", rep("c", ncol(.))), 
        digits = c(0, 0, rep(2, 7), 0)) %>%
    add_header_above(c(" " = 2, "POS" = 1, "NER"= 5, "Dependency Parsing" = 2, "Speed" = 1)) %>%
    highlight_highest(., data, columns = high_columns) %>%
    kable_styling(latex_options = c( "scale_down")) %>% 
    footnote(number = c("Highest scores are in bold and second highest is underlined.", 
                         "Empty cells denote that the particular model does not contain a trained component for the specific task.", 
                         "WPS indicate words pr. second. Only reported to DaCy models."))
  return(res)
}

performance = performance %>% 
  select(Framework, Model, Accuracy, PER = Person, LOC=Location, ORG=Organization, MISC=Misc, `Avg. F1` = `Avg F1`, UAS, LAS, WPS) 
high = colnames(performance)[3:length(colnames(performance))]
performance %>% 
  make_table(data = .) #%>% 
  save_kable(file ="../../img/perf.png")

performance %>% make_table(data = .) %>% 
  save_kable(file ="../../DaCy_training/img/perf_training.png")
```


