---
title: "Bootstrap CI"
output: html_document
---

### DeepYeast

```{r, echo=F, message=F}
library(dplyr)
library(reshape2)
library(knitr)

PATH = "/home/hpc_tanel/deep-yeast/make-data/data/HOwt/"
VAL_PRED_PATH = "/storage/hpc_tanel/features/HOwt_png_vgg_A_bn_final_prob_val.txt"
TEST_PRED_PATH = "/storage/hpc_tanel/features/HOwt_png_vgg_A_bn_final_prob_test.txt"
MODEL_NAME = "DeepYeast"
```


```{r, echo=F}
doc = read.table(paste0(PATH, "HOwt_doc.txt"), sep=";", header=T, as.is=T)

val = read.table(paste0(PATH, "HOwt_val.txt"), as.is=T)
names(val) = c("src", "y")
val = left_join(val, doc, by=c("y"="label_idx"))
tmp = read.table(VAL_PRED_PATH, as.is=T)
preds = apply(tmp, 1, which.max) - 1
val$pred = preds
val$pred_label = doc$label[preds + 1]

test = read.table(paste0(PATH, "HOwt_test.txt"), as.is=T)
names(test) = c("src", "y")
test = left_join(test, doc, by=c("y"="label_idx"))
tmp = read.table(TEST_PRED_PATH, as.is=T)
preds = apply(tmp, 1, which.max) - 1
test$pred = preds
test$pred_label = doc$label[preds + 1]

```


```{r, echo=F, fig.width=10, fig.height=7, message=F}
set.seed(1235)

recall_tbl = test %>%
  group_by(label) %>%
  summarise(#num_imgs=n(),
            #num_correct=sum(y==pred),
            recall=mean(y==pred))
recall_tbl$boot_mean = rep(NA, nrow(recall_tbl))
recall_tbl$boot_025 = rep(NA, nrow(recall_tbl))
recall_tbl$boot_975 = rep(NA, nrow(recall_tbl))

precision_tbl = test %>%
  group_by(pred_label) %>%
  summarise(precision=mean(y==pred))
precision_tbl$boot_mean = rep(NA, nrow(precision_tbl))
precision_tbl$boot_025 = rep(NA, nrow(precision_tbl))
precision_tbl$boot_975 = rep(NA, nrow(precision_tbl))

NUM_ROUNDS = 20000

for(label in unique(test$label)){
  sub = test[test$label == label, ]
  recalls = rep(NA, NUM_ROUNDS)
  
  for(i in 1:NUM_ROUNDS){
    boot_idx = sample(1:nrow(sub), nrow(sub), replace = T)
    boot = sub[boot_idx, ]
    
    recall = mean(boot$y==boot$pred)
    recalls[i] = recall
  }
  #print(label)
  q025 = quantile(recalls, 0.025)
  q975 = quantile(recalls, 0.975)
  #print(paste0("Recall: ", mean(recalls), " - (", q025, ", ", q975, ")"))
  
  recall_tbl$boot_mean[recall_tbl$label==label] = mean(recalls)
  recall_tbl$boot_025[recall_tbl$label==label] = q025
  recall_tbl$boot_975[recall_tbl$label==label] = q975
}

for(label in unique(test$label)){
  sub = test[test$pred_label == label, ]
  precisions = rep(NA, NUM_ROUNDS)
  
  for(i in 1:NUM_ROUNDS){
    boot_idx = sample(1:nrow(sub), nrow(sub), replace = T)
    boot = sub[boot_idx, ]
    
    precision = mean(boot$y==boot$pred)
    precisions[i] = precision
  }
  
  #print(label)
  q025 = quantile(precisions, 0.025)
  q975 = quantile(precisions, 0.975)
  #print(paste0("Precision: ", mean(precisions), " - (", q025, ", ", q975, ")"))
  
  precision_tbl$boot_mean[precision_tbl$pred_label==label] = mean(precisions)
  precision_tbl$boot_025[precision_tbl$pred_label==label] = q025
  precision_tbl$boot_975[precision_tbl$pred_label==label] = q975
}

kable(recall_tbl, digits=3)
kable(precision_tbl, digits=3)

write.csv(recall_tbl, file="recall_boot_CNN_tbl.csv", row.names = F)
write.csv(precision_tbl, file="precision_boot_CNN_tbl.csv", row.names = F)
```

### RF

```{r, echo=F}
PATH = "/home/hpc_tanel/deep-yeast/make-data/data/HOwt/"
VAL_PRED_PATH = "../random-forest/rf_preds_val.csv"
TEST_PRED_PATH = "../random-forest/rf_preds_test.csv"
MODEL_NAME = "Random forest"

doc = read.table(paste0(PATH, "HOwt_doc.txt"), sep=";", header=T, as.is=T)

val = read.table(paste0(PATH, "HOwt_val.txt"), as.is=T)
names(val) = c("src", "y")
val = left_join(val, doc, by=c("y"="label_idx"))
val$pred = read.table(VAL_PRED_PATH, as.is=T)[, 1]
val$pred_label = doc$label[val$pred + 1]

test = read.table(paste0(PATH, "HOwt_test.txt"), as.is=T)
names(test) = c("src", "y")
test = left_join(test, doc, by=c("y"="label_idx"))
test$pred = read.table(TEST_PRED_PATH, as.is=T)[, 1]
test$pred_label = doc$label[test$pred + 1]

```

```{r, echo=F, fig.width=10, fig.height=7, message=F}
set.seed(1235)

recall_tbl = test %>%
  group_by(label) %>%
  summarise(#num_imgs=n(),
            #num_correct=sum(y==pred),
            recall=mean(y==pred))
recall_tbl$boot_mean = rep(NA, nrow(recall_tbl))
recall_tbl$boot_025 = rep(NA, nrow(recall_tbl))
recall_tbl$boot_975 = rep(NA, nrow(recall_tbl))

precision_tbl = test %>%
  group_by(pred_label) %>%
  summarise(precision=mean(y==pred))
precision_tbl$boot_mean = rep(NA, nrow(precision_tbl))
precision_tbl$boot_025 = rep(NA, nrow(precision_tbl))
precision_tbl$boot_975 = rep(NA, nrow(precision_tbl))

NUM_ROUNDS = 20000

for(label in unique(test$label)){
  sub = test[test$label == label, ]
  recalls = rep(NA, NUM_ROUNDS)
  
  for(i in 1:NUM_ROUNDS){
    boot_idx = sample(1:nrow(sub), nrow(sub), replace = T)
    boot = sub[boot_idx, ]
    
    recall = mean(boot$y==boot$pred)
    recalls[i] = recall
  }
  #print(label)
  q025 = quantile(recalls, 0.025)
  q975 = quantile(recalls, 0.975)
  #print(paste0("Recall: ", mean(recalls), " - (", q025, ", ", q975, ")"))
  
  recall_tbl$boot_mean[recall_tbl$label==label] = mean(recalls)
  recall_tbl$boot_025[recall_tbl$label==label] = q025
  recall_tbl$boot_975[recall_tbl$label==label] = q975
}

for(label in unique(test$label)){
  sub = test[test$pred_label == label, ]
  precisions = rep(NA, NUM_ROUNDS)
  
  for(i in 1:NUM_ROUNDS){
    boot_idx = sample(1:nrow(sub), nrow(sub), replace = T)
    boot = sub[boot_idx, ]
    
    precision = mean(boot$y==boot$pred)
    precisions[i] = precision
  }
  
  #print(label)
  q025 = quantile(precisions, 0.025)
  q975 = quantile(precisions, 0.975)
  #print(paste0("Precision: ", mean(precisions), " - (", q025, ", ", q975, ")"))
  
  precision_tbl$boot_mean[precision_tbl$pred_label==label] = mean(precisions)
  precision_tbl$boot_025[precision_tbl$pred_label==label] = q025
  precision_tbl$boot_975[precision_tbl$pred_label==label] = q975
}

kable(recall_tbl, digits=3)
kable(precision_tbl, digits=3)

write.csv(recall_tbl, file="recall_boot_RF_tbl.csv", row.names = F)
write.csv(precision_tbl, file="precision_boot_RF_tbl.csv", row.names = F)
```
