keywords_analysis.qmd

---
title: "# CODES - A GUIDE TO MISINFORMATION DETECTION DATASET"
format: pdf
editor: visual
---

# Data

```{r, message=FALSE, warning=FALSE}

# Librairies --------------------------------------------------------------

library(readr)
library(tidyverse)
library(randomForest)
library(caret)
library(stringr)
library(tidytext)
library(quanteda)
library(jsonlite)
library(xlsx)
library(rdflib)
library(nnet)
library(broom)
library(ggrepel)
library(tm) 
library(yardstick)
library(MLmetrics)
library(viridis)
library(patchwork)

# Data - Claims --------------------------------------------------------------------

dat_claims <- read_csv("C:/Users/client/Desktop/datasets/all_data/dat_claims.csv")

# See all datasets --------------------------------------------------------

dat_claims %>% 
  select(dataset) %>% 
  table()

```

# Labels

```{r, warning=FALSE}

# Veracity count - Label Disparity per Dataset ----------------------------

datasets <- unique(dat_claims$dataset)

# List for results
results_table <- list()
results_prop <- list()

# Loop
for (dataset_name in datasets) {
  current_data <- dat_claims %>% 
    filter(dataset == dataset_name) %>% 
    select(where(~ !all(is.na(.))))
  
  result_table <- table(current_data$veracity)
  results_table[[dataset_name]] <- result_table
  
  result_prop <- prop.table(result_table) * 100
  results_prop[[dataset_name]] <- result_prop
}

# Results
results_table
results_prop


```

# Keywords analysis

```{r}
# 40 most frequently used words by database

datasets_list <- unique(dat_claims$dataset)

for (dataset in datasets_list) {
  result <- dat_claims %>%
    filter(dataset == !!dataset, !is.na(claim)) %>%
    mutate(claim = str_to_lower(claim)) %>%
    unnest_tokens(word, claim) %>%
    filter(!word %in% stop_words$word) %>%
    group_by(dataset, veracity, word) %>%
    count(name = "n") %>%
    ungroup() %>%
    group_by(word) %>%
    summarize(total = sum(n), .groups = 'drop') %>%
    arrange(desc(total)) %>%
    slice_head(n = 40) 
  
  result_with_veracity <- dat_claims %>%
    filter(dataset == !!dataset, !is.na(claim)) %>%
    mutate(claim = str_to_lower(claim)) %>%
    unnest_tokens(word, claim) %>%
    filter(!word %in% stop_words$word) %>%
    group_by(dataset, veracity, word) %>%
    count(name = "n") %>%
    filter(word %in% result$word) %>%
    ungroup()
  
  assign(paste0("frequency_", gsub(" ", "_", dataset)), 
         result_with_veracity, envir = .GlobalEnv)}

ls(pattern = "^frequency_")
```

## Check-COVID

### Keywords Predictivity T-F

```{r, warning=FALSE}
checkcovid_filtered <- dat_claims %>%
  filter(dataset == "checkcovid") %>%
  filter(veracity != 3)

frequency_checkcovid_unique <- frequency_checkcovid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_checkcovid_unique$word

corpus <- Corpus(VectorSource(checkcovid_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(checkcovid_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest - Filtered):")
print(confusion_matrix)

# F1 Macro

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### Keywords Predictivity T-F-M

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

checkcovid <- dat_claims %>%
  filter(dataset == "checkcovid") %>% 
  filter(!veracity %in% c(3, 9))

frequency_checkcovid_unique <- frequency_checkcovid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_checkcovid_unique$word

corpus <- Corpus(VectorSource(checkcovid$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(checkcovid$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))
```

## ClaimsKG

### T-F

```{r, message=FALSE, warning=FALSE}

# 2 labels ----------------------------------------------------------------

claimskg <- dat_claims %>%
  filter(dataset == "claimskg") %>%
  filter(!veracity %in% c(3, 9))

frequency_claimskg_unique <- frequency_claimskg %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_claimskg_unique$word

corpus <- Corpus(VectorSource(claimskg$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(claimskg$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score for random predictions
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-M

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

claimskg <- dat_claims %>%
  filter(dataset == "claimskg") %>% 
  filter(veracity != 9)

frequency_claimskg_unique <- frequency_claimskg %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_claimskg_unique$word

corpus <- Corpus(VectorSource(claimskg$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(claimskg$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## Climate-Fever

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

climate_fever_filtered <- dat_claims %>%
  filter(dataset == "climate_fever") %>%
  filter(!veracity %in% c(3, 9))

frequency_climate_fever_unique <- frequency_climate_fever %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_climate_fever_unique$word

corpus <- Corpus(VectorSource(climate_fever_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(climate_fever_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train,
                         ntree = 100, 
                         maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-M

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

climate_fever <- dat_claims %>%
  filter(dataset == "climate_fever") %>% 
  filter(veracity != 9)

frequency_climate_fever_unique <- frequency_climate_fever %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_climate_fever_unique$word

corpus <- Corpus(VectorSource(climate_fever$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(climate_fever$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train,
                         ntree = 100, 
                         maxnodes = 20)

# Step 6: Make 
predictions <- predict(rf_model, 
                       X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## CoAID

### T-F

```{r, warning=FALSE}

# Coaid - 2 labels --------------------------------------------------------

coaid <- dat_claims %>% 
  filter(dataset == "coaid") %>% 
  filter(!veracity %in% c(3, 9))

frequency_coaid_unique <- frequency_coaid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_coaid_unique$word

corpus <- Corpus(VectorSource(coaid$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(coaid$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score:", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

COAID has no third label (Mixed)

## Covid-19-rumor

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

covid_19_rumor_filtered <- dat_claims %>%
  filter(dataset == "covid_19_rumor") %>%
  filter(!veracity %in% c(3, 9))

frequency_covid_19_rumor_unique <- frequency_covid_19_rumor %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_covid_19_rumor_unique$word

corpus <- Corpus(VectorSource(covid_19_rumor_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(covid_19_rumor_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-M - No label Mixed

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

covid_19_rumor <- dat_claims %>%
  filter(dataset == "covid_19_rumor") %>% 
  filter(veracity != 9)

frequency_covid_19_rumor_unique <- frequency_covid_19_rumor %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_covid_19_rumor_unique$word

corpus <- Corpus(VectorSource(covid_19_rumor$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(covid_19_rumor$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## Covid-Fact

### T-F

```{r, warning=FALSE}

covidfact <- dat_claims %>%
  filter(dataset == "covidfact") %>%
  filter(!veracity %in% c(3, 9))

frequency_covidfact_unique <- frequency_covidfact %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_covidfact_unique$word

corpus <- Corpus(VectorSource(covidfact$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(covidfact$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## DeFakts

### T-F

```{r, warning=FALSE}
# 2 labels, has not "mixed"
defakts <- dat_claims %>%
  filter(dataset == "defakts") %>% 
  filter(!veracity %in% c(3, 9))

frequency_defakts_unique <- frequency_defakts %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_defakts_unique$word

corpus <- Corpus(VectorSource(defakts$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("de")) # German language
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(defakts$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## FakeCovid

### T-F

```{r, warning=FALSE}

fakecovid_filtered <- dat_claims %>%
  filter(dataset == "fakecovid") %>%
  filter(!veracity %in% c(3, 9))

frequency_fakecovid_unique <- frequency_fakecovid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_fakecovid_unique$word

corpus <- Corpus(VectorSource(fakecovid_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(fakecovid_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

# F1 macro score
f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  

  if (TP == 0) {
    return(0)  
  }
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  return(2 * (precision * recall) / (precision + recall))
})

macro_f1_random <- mean(f1_scores_random, na.rm = TRUE)
print(paste("Macro F1 Score (random forest):", macro_f1_random))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

# F1 macro score
f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  if (TP == 0) {
    return(0) 
  }
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  return(2 * (precision * recall) / (precision + recall))
})

macro_f1_random <- mean(f1_scores_random, na.rm = TRUE)
print(paste("Macro F1 Score (random prediction) :", macro_f1_random))
```

## T-F-M

```{r, warning=FALSE}

fakecovid <- dat_claims %>%
  filter(dataset == "fakecovid") %>% 
  filter(veracity != 9)

frequency_fakecovid_unique <- frequency_fakecovid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_fakecovid_unique$word

corpus <- Corpus(VectorSource(fakecovid$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(fakecovid$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))


# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

# F1 macro score
f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  if (TP == 0) {
    return(0) 
  }
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  return(2 * (precision * recall) / (precision + recall))
})

macro_f1_random <- mean(f1_scores_random, na.rm = TRUE)
print(paste("Macro F1 Score (random prediction):", macro_f1_random))
```

## FAVIQ

### T-F

```{r, warning=FALSE}

faviq <- dat_claims %>%
  filter(dataset == "faviq") %>% # has no other
  filter(!veracity %in% c(3, 9))

frequency_faviq_unique <- frequency_faviq %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_faviq_unique$word

corpus <- Corpus(VectorSource(faviq$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(faviq$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## FEVER

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

fever_filtered <- dat_claims %>%
  filter(dataset == "fever") %>%
  filter(!veracity %in% c(3, 9))

frequency_fever_unique <- frequency_fever %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_fever_unique$word

corpus <- Corpus(VectorSource(fever_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(fever_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## FEVEROUS

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

feverous_filtered <- dat_claims %>%
  filter(dataset == "feverous") %>%
  filter(!veracity %in% c(3, 9))

frequency_feverous_unique <- frequency_feverous %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_feverous_unique$word

corpus <- Corpus(VectorSource(feverous_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(feverous_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## FibVID

### T-F

```{r, warning=FALSE}

fibvid <- dat_claims %>% # Has only two labels
  filter(dataset == "fibvid") %>% 
  filter(!veracity %in% c(3, 9))

frequency_fibvid_unique <- frequency_fibvid %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_fibvid_unique$word

corpus <- Corpus(VectorSource(fibvid$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(fibvid$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)

  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score for random predictions
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))
```

## HoVer

### T-F

```{r, warning=FALSE}

hover <- dat_claims %>% # Has only two labels T-F
  filter(dataset == "hover") %>% 
  filter(!veracity %in% c(3, 9))

frequency_hover_unique <- frequency_hover %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_hover_unique$word

corpus <- Corpus(VectorSource(hover$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(hover$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)

  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }

  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }

  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))
```

## IFND

### T-F

```{r, warning=FALSE}

IFND <- dat_claims %>%
  filter(dataset == "IFND") %>% # Only two labels in the dataset
  filter(!veracity %in% c(3, 9))

frequency_IFND_unique <- frequency_IFND %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_IFND_unique$word

corpus <- Corpus(VectorSource(IFND$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(IFND$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, 
                                 Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)

  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))


```

## LIAR

## T-F

```{r, warning=FALSE}

liar <- dat_claims %>%
  filter(dataset == "liar") %>% 
  filter(!veracity %in% c(3, 9))

frequency_liar_unique <- frequency_liar %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_liar_unique$word

corpus <- Corpus(VectorSource(liar$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(liar$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## T-F-M

```{r}
liar <- dat_claims %>%
  filter(dataset == "liar") %>% 
  filter(veracity != 9)

frequency_liar_unique <- frequency_liar %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_liar_unique$word

corpus <- Corpus(VectorSource(liar$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(liar$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))


# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

# F1 macro score
f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  if (TP == 0) {
    return(0) 
  }
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  return(2 * (precision * recall) / (precision + recall))
})

macro_f1_random <- mean(f1_scores_random, na.rm = TRUE)
print(paste("Macro F1 Score (random prediction):", macro_f1_random))
```

## LIAR-NEW

### T-F

```{r, warning=FALSE}

liar_new <- dat_claims %>%
  filter(dataset == "liar_new") %>% 
  filter(!veracity %in% c(3, 9))

frequency_liar_new_unique <- frequency_liar_new %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_liar_new_unique$word

corpus <- Corpus(VectorSource(liar_new$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(liar_new$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, 
                                   list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train, 
                         ntree = 100, 
                         maxnodes = 20)

predictions <- predict(rf_model, 
                       X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))
```

### T-F-M

```{r}
liar_new <- dat_claims %>%
  filter(dataset == "liar_new") %>% 
  filter(veracity != 9)

frequency_liar_new_unique <- frequency_liar_new %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_liar_new_unique$word

corpus <- Corpus(VectorSource(liar_new$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(liar_new$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))


# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

# F1 macro score
f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  if (TP == 0) {
    return(0) 
  }
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  return(2 * (precision * recall) / (precision + recall))
})

macro_f1_random <- mean(f1_scores_random, na.rm = TRUE)
print(paste("Macro F1 Score (random prediction):", macro_f1_random))
```

## MM-COVID

## T-F

```{r, warning=FALSE}

## Has 2 labels

mm_covid <- dat_claims %>%
  filter(dataset == "mm-covid") %>% 
  filter(!veracity %in% c(3, 9))

frequency_mm_covid_unique <- `frequency_mm-covid` %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_mm_covid_unique$word

corpus <- Corpus(VectorSource(mm_covid$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(mm_covid$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## NLP4IF-2021

### T-F

```{r, warning=FALSE}

## Dataset
nlp4if <- dat_claims %>%
  filter(dataset == "nlp4if") %>%  # Two labels 
  filter(!veracity %in% c(3, 9))

## Top 40 keywords
frequency_nlp4if_unique <- frequency_nlp4if %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_nlp4if_unique$word

## Corpus
corpus <- Corpus(VectorSource(nlp4if$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(nlp4if$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

## Split - Training and Test
set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

## Predictions
predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

## F1 Score
f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score - With frenquency of labels
# ----------------------

## Addind frequency of each class in the random forest classifier
class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, 
                                 Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## PubHealthTab

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

pubhealthtab <- dat_claims %>% 
  filter(dataset == "pubhealthtab") %>% 
  filter(!is.na(claim) & claim != "") %>% 
  filter(!veracity %in% c(3, 9))

frequency_pubhealthtab_unique <- frequency_pubhealthtab %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_pubhealthtab_unique$word

corpus <- Corpus(VectorSource(pubhealthtab$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(pubhealthtab$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, 
                                   list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train, 
                         ntree = 100, 
                         maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, 
                          Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score:", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))


```

### 

## Rumors

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

rumors_filtered <- dat_claims %>%
  filter(dataset == "rumors") %>%
  filter(!veracity %in% c(3, 9))

frequency_rumors_unique <- frequency_rumors %>%
  distinct(word, .keep_all = TRUE)

top_keywords_rumors <- frequency_rumors_unique$word

corpus_rumors <- Corpus(VectorSource(rumors_filtered$claim))
corpus_rumors <- tm_map(corpus_rumors, content_transformer(tolower))
corpus_rumors <- tm_map(corpus_rumors, removePunctuation)
corpus_rumors <- tm_map(corpus_rumors, removeNumbers)
corpus_rumors <- tm_map(corpus_rumors, removeWords, stopwords("en"))
corpus_rumors <- tm_map(corpus_rumors, stripWhitespace)

dtm_rumors <- DocumentTermMatrix(corpus_rumors, 
                                 control = list(dictionary = top_keywords_rumors, 
                                                               binary = TRUE))
dtm_matrix_rumors <- as.matrix(dtm_rumors)

keyword_df_rumors <- as.data.frame(dtm_matrix_rumors)

X_rumors <- keyword_df_rumors
Y_rumors <- as.factor(rumors_filtered$veracity)

valid_indices_rumors <- which(!is.na(Y_rumors))
X_rumors <- X_rumors[valid_indices_rumors, ]
Y_rumors <- Y_rumors[valid_indices_rumors]

set.seed(42)
train_index_rumors <- createDataPartition(Y_rumors, p = 0.75, list = FALSE)
X_train_rumors <- X_rumors[train_index_rumors, ]
Y_train_rumors <- Y_rumors[train_index_rumors]
X_test_rumors <- X_rumors[-train_index_rumors, ]
Y_test_rumors <- Y_rumors[-train_index_rumors]

rf_model_rumors <- randomForest(X_train_rumors, Y_train_rumors, ntree = 100, maxnodes = 20)

predictions_rumors <- predict(rf_model_rumors, X_test_rumors)

confusion_matrix_rumors <- table(predictions_rumors, Y_test_rumors)
print("Confusion Matrix (Random Forest - Rumors):")
print(confusion_matrix_rumors)

f1_scores_rumors <- sapply(levels(Y_test_rumors), function(class) {
  TP <- confusion_matrix_rumors[class, class]
  FP <- sum(confusion_matrix_rumors[, class]) - TP
  FN <- sum(confusion_matrix_rumors[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_rumors <- mean(f1_scores_rumors)
print(paste("Macro F1 Score (Random Forest - Rumors):", macro_f1_rumors))

# ----------------------
# Calculate the baseline F1-score with random predictions for rumors
# ----------------------

class_distribution_rumors <- prop.table(table(Y_rumors))
random_predictions_rumors <- sample(levels(Y_rumors), 
                                    size = length(Y_test_rumors), 
                                    replace = TRUE, 
                                    prob = class_distribution_rumors)

confusion_matrix_random_rumors <- table(random_predictions_rumors, Y_test_rumors)

f1_scores_random_rumors <- sapply(levels(Y_test_rumors), function(class) {
  TP <- confusion_matrix_random_rumors[class, class]
  FP <- sum(confusion_matrix_random_rumors[, class]) - TP
  FN <- sum(confusion_matrix_random_rumors[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random_rumors <- mean(f1_scores_random_rumors)
print(paste("Macro F1 Score (Random Predictions - Rumors):", macro_f1_random_rumors))

```

### T-F-O

```{r, warning=FALSE}
# 3 labels ----------------------------------------------------------------

rumors_filtered_no_filter <- dat_claims %>%
  filter(dataset == "rumors") %>% 
  filter(veracity != 9)

frequency_rumors_unique <- frequency_rumors %>%
  distinct(word, .keep_all = TRUE)

top_keywords_rumors_no_filter <- frequency_rumors_unique$word

corpus_rumors_no_filter <- Corpus(VectorSource(rumors_filtered_no_filter$claim))
corpus_rumors_no_filter <- tm_map(corpus_rumors_no_filter, content_transformer(tolower))
corpus_rumors_no_filter <- tm_map(corpus_rumors_no_filter, removePunctuation)
corpus_rumors_no_filter <- tm_map(corpus_rumors_no_filter, removeNumbers)
corpus_rumors_no_filter <- tm_map(corpus_rumors_no_filter, removeWords, stopwords("en"))
corpus_rumors_no_filter <- tm_map(corpus_rumors_no_filter, stripWhitespace)

dtm_rumors_no_filter <- DocumentTermMatrix(corpus_rumors_no_filter, control = list(dictionary = top_keywords_rumors_no_filter, 
                                                                                   binary = TRUE))
dtm_matrix_rumors_no_filter <- as.matrix(dtm_rumors_no_filter)

keyword_df_rumors_no_filter <- as.data.frame(dtm_matrix_rumors_no_filter)

X_rumors_no_filter <- keyword_df_rumors_no_filter
Y_rumors_no_filter <- as.factor(rumors_filtered_no_filter$veracity)

valid_indices_rumors_no_filter <- which(!is.na(Y_rumors_no_filter))
X_rumors_no_filter <- X_rumors_no_filter[valid_indices_rumors_no_filter, ]
Y_rumors_no_filter <- Y_rumors_no_filter[valid_indices_rumors_no_filter]

set.seed(42)
train_index_rumors_no_filter <- createDataPartition(Y_rumors_no_filter, p = 0.75, list = FALSE)
X_train_rumors_no_filter <- X_rumors_no_filter[train_index_rumors_no_filter, ]
Y_train_rumors_no_filter <- Y_rumors_no_filter[train_index_rumors_no_filter]
X_test_rumors_no_filter <- X_rumors_no_filter[-train_index_rumors_no_filter, ]
Y_test_rumors_no_filter <- Y_rumors_no_filter[-train_index_rumors_no_filter]

rf_model_rumors_no_filter <- randomForest(X_train_rumors_no_filter, Y_train_rumors_no_filter, ntree = 100, maxnodes = 20)

predictions_rumors_no_filter <- predict(rf_model_rumors_no_filter, X_test_rumors_no_filter)

confusion_matrix_rumors_no_filter <- table(predictions_rumors_no_filter, Y_test_rumors_no_filter)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix_rumors_no_filter)

f1_scores_rumors_no_filter <- sapply(levels(Y_test_rumors_no_filter), function(class) {
  TP <- confusion_matrix_rumors_no_filter[class, class]
  FP <- sum(confusion_matrix_rumors_no_filter[, class]) - TP
  FN <- sum(confusion_matrix_rumors_no_filter[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  # NaN handling
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  # F1 Score
  2 * (precision * recall) / (precision + recall)
})

macro_f1_rumors_no_filter <- mean(f1_scores_rumors_no_filter)
print(paste("Macro F1 Score (Random Forest):", 
            macro_f1_rumors_no_filter))

# ----------------------
# Calculate the baseline F1-score with random predictions for rumors without filtering
# ----------------------

class_distribution_rumors_no_filter <- prop.table(table(Y_rumors_no_filter))
random_predictions_rumors_no_filter <- sample(levels(Y_rumors_no_filter), 
                                              size = length(Y_test_rumors_no_filter), 
                                              replace = TRUE, 
                                              prob = class_distribution_rumors_no_filter)

confusion_matrix_random_rumors_no_filter <- table(factor(random_predictions_rumors_no_filter, levels = levels(Y_test_rumors_no_filter)), 
                                                  factor(Y_test_rumors_no_filter, levels = levels(Y_test_rumors_no_filter)))


f1_scores_random_rumors_no_filter <- sapply(levels(Y_test_rumors_no_filter), function(class) {
  TP <- confusion_matrix_random_rumors_no_filter[class, class]
  FP <- sum(confusion_matrix_random_rumors_no_filter[, class]) - TP
  FN <- sum(confusion_matrix_random_rumors_no_filter[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random_rumors_no_filter <- mean(f1_scores_random_rumors_no_filter)
print(paste("Macro F1 Score (Random Predictions):", 
            macro_f1_random_rumors_no_filter))

```

## Snopes

## T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

snopes_filtered <- dat_claims %>%
  filter(dataset == "snopes") %>%
  filter(!veracity %in% c(3, 9))

frequency_snopes_unique <- frequency_snopes %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_snopes_unique$word

corpus <- Corpus(VectorSource(snopes_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(snopes_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-O

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

snopes <- dat_claims %>%
  filter(dataset == "snopes") %>% 
  filter(veracity != 9)

frequency_snopes_unique <- frequency_snopes %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_snopes_unique$word

corpus <- Corpus(VectorSource(snopes$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(snopes$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## Truthseeker2023

```{r, warning=FALSE}

truthseeker2023 <- dat_claims %>%
  filter(dataset == "truthseeker2023") %>% # Has only two labels
  filter(!veracity %in% c(3, 9))

frequency_truthseeker2023_unique <- frequency_truthseeker2023 %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_truthseeker2023_unique$word

corpus <- Corpus(VectorSource(truthseeker2023$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(truthseeker2023$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## Twitter15

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

twitter15 <- dat_claims %>% 
  filter(dataset == "twitter15") %>% 
  filter(!is.na(claim) & claim != "") %>% 
  filter(!veracity %in% c(3, 9))

# Top 40 keywords
frequency_twitter15_unique <- frequency_twitter15 %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_twitter15_unique$word

corpus <- Corpus(VectorSource(twitter15$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(twitter15$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, 
                                   list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train, 
                         ntree = 100, 
                         maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, 
                          Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)

print(paste("Macro F1 Score (random forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), 
                             replace = TRUE, prob = class_distribution)
confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-0

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

twitter15 <- dat_claims %>% 
  filter(dataset == "twitter15") %>% 
  filter(!is.na(claim) & claim != "") %>% 
  filter(veracity != 9)

frequency_twitter15_unique <- frequency_twitter15 %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_twitter15_unique$word

corpus <- Corpus(VectorSource(twitter15$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(twitter15$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, 
                                   list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, 
                         Y_train, 
                         ntree = 100, 
                         maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, 
                          Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)

# F1 Score macro
print(paste("Macro F1 Score (random forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), 
                             size = length(Y_test), 
                             replace = TRUE, 
                             prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  # F1 Score
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))


```

## Twitter16

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

twitter16 <- dat_claims %>% 
  filter(dataset == "twitter16") %>% 
  filter(!is.na(claim) & claim != "") %>% 
  filter(!veracity %in% c(3, 9))

frequency_twitter16_unique <- frequency_twitter16 %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_twitter16_unique$word

corpus <- Corpus(VectorSource(twitter16$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(twitter16$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)

print(paste("Macro F1 Score (random forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))


```

### T-F-O

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

twitter16 <- dat_claims %>% 
  filter(dataset == "twitter16") %>% 
  filter(!is.na(claim) & claim != "") %>% 
  filter(veracity != 9)

frequency_twitter16_unique <- frequency_twitter16 %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_twitter16_unique$word

corpus <- Corpus(VectorSource(twitter16$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(twitter16$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0)}
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)

print(paste("Macro F1 Score:", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions - Unfiltered):", macro_f1_random))

```

## VERITE

### T-F

```{r, warning=FALSE}

verite <- dat_claims %>%
  filter(dataset == "verite") %>% # has only 2 labels
  filter(!veracity %in% c(3, 9))

frequency_verite_unique <- frequency_verite %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_verite_unique$word

corpus <- Corpus(VectorSource(verite$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(verite$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score for random predictions
macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

## X-Fact

### T-F

```{r, warning=FALSE}

# 2 labels ----------------------------------------------------------------

x_fact_filtered <- dat_claims %>%
  filter(dataset == "x_fact") %>%
  filter(!veracity %in% c(3, 9))

frequency_x_fact_unique <- frequency_x_fact %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_x_fact_unique$word

corpus <- Corpus(VectorSource(x_fact_filtered$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(x_fact_filtered$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

### T-F-O

```{r, warning=FALSE}

# 3 labels ----------------------------------------------------------------

x_fact <- dat_claims %>%
  filter(dataset == "x_fact") %>% 
  filter(veracity != 9)

frequency_x_fact_unique <- frequency_x_fact %>%
  distinct(word, .keep_all = TRUE)

top_keywords <- frequency_x_fact_unique$word

corpus <- Corpus(VectorSource(x_fact$claim))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, removeWords, stopwords("en"))
corpus <- tm_map(corpus, stripWhitespace)

dtm <- DocumentTermMatrix(corpus, control = list(dictionary = top_keywords, 
                                                 binary = TRUE))
dtm_matrix <- as.matrix(dtm)

keyword_df <- as.data.frame(dtm_matrix)

X <- keyword_df
Y <- as.factor(x_fact$veracity)

valid_indices <- which(!is.na(Y))
X <- X[valid_indices, ]
Y <- Y[valid_indices]

set.seed(42)
train_index <- createDataPartition(Y, p = 0.75, list = FALSE)
X_train <- X[train_index, ]
Y_train <- Y[train_index]
X_test <- X[-train_index, ]
Y_test <- Y[-train_index]

rf_model <- randomForest(X_train, Y_train, ntree = 100, maxnodes = 20)

predictions <- predict(rf_model, X_test)

confusion_matrix <- table(predictions, Y_test)
print("Confusion Matrix (Random Forest):")
print(confusion_matrix)

f1_scores <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix[class, class]
  FP <- sum(confusion_matrix[, class]) - TP
  FN <- sum(confusion_matrix[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

# Macro F1 Score
macro_f1 <- mean(f1_scores)
print(paste("Macro F1 Score (Random Forest):", macro_f1))

# ----------------------
# Calculate the baseline F1-score with random predictions
# ----------------------

class_distribution <- prop.table(table(Y))
random_predictions <- sample(levels(Y), size = length(Y_test), replace = TRUE, prob = class_distribution)

confusion_matrix_random <- table(random_predictions, Y_test)

f1_scores_random <- sapply(levels(Y_test), function(class) {
  TP <- confusion_matrix_random[class, class]
  FP <- sum(confusion_matrix_random[, class]) - TP
  FN <- sum(confusion_matrix_random[class, ]) - TP
  
  precision <- TP / (TP + FP)
  recall <- TP / (TP + FN)
  
  if (is.nan(precision) || is.nan(recall)) {
    return(0) 
  }
  
  2 * (precision * recall) / (precision + recall)
})

macro_f1_random <- mean(f1_scores_random)
print(paste("Macro F1 Score (Random Predictions):", macro_f1_random))

```

# Graph

```{r}
# GRAPH - KEYWORDS ANALYSIS -----------------------------------------------

## IFND

IFND <- dat_claims %>%
  filter(dataset == "IFND") %>%
  mutate(claim_text = str_to_lower(claim)) %>%
  unnest_tokens(word, claim) %>%
  anti_join(stop_words)

word_freq <- IFND %>%
  count(veracity, word) %>%
  group_by(word) %>%
  mutate(freq = n / sum(n))

word_freq <- word_freq %>%
  filter(n >= 10)

word_freq_clean <- word_freq %>%
  filter(!is.na(word), word != "url")

# Top 40 words
top_40_words <- word_freq_clean %>%
  group_by(word) %>%
  summarize(total_n = sum(n)) %>% 
  arrange(desc(total_n)) %>%
  slice_head(n = 40)

word_freq_top_40 <- word_freq_clean %>%
  filter(word %in% top_40_words$word)

# Mean frequency
mean_freq <- word_freq_top_40 %>%
  pull(freq) %>% 
  mean(na.rm = TRUE)

mean_freq

### Graph

ifnd_graph <- ggplot(word_freq_top_40, aes(x = freq, 
                            y = veracity, 
                            label = word)) +
  geom_point(size = 1) +
  geom_text_repel(color = "black",
                  size = 3, 
                  nudge_x = 0.03,
                  nudge_y = 0.1,
                  box.padding = 0.4,
                  max.overlaps = 50,
                  segment.size = 0.3,
                  segment.color = "grey") +
  labs(x = "Word Frequency (%)",
       y = "Veracity") +
  theme_minimal() +
  theme(plot.title = element_text(hjust = 0.5, size = 14),
        plot.subtitle = element_text(hjust = 0.5, size = 12), 
        panel.grid.minor.x = element_blank(),
        panel.grid.minor.y = element_blank(),
        axis.text = element_text(size = 6),
        axis.title = element_text(size = 10),
        axis.text.x = element_text(size = 8),
        axis.text.y = element_text(size = 8),
        axis.line = element_line(color = "gray47"), 
        axis.ticks = element_line(color = "gray47")) +
  scale_x_continuous(limits = c(0.59, 1.1), 
                     breaks = seq(0.6, 1, by = 0.1),  
                     expand = expansion(add = c(0, 0.1))) +  
   scale_y_reverse(limits = c(2.05, 0.5),
                   breaks = c(1, 2),
                   labels = c("True", "False"))

ifnd_graph

### Save
ggsave(ifnd_graph,
       filename = "ifnd_graph.png",
       dpi = 320,
       bg = "white",
       units = "cm",
       height = 12,
       width = 18)


# mm-covid ----------------------------------------------------------------

mmcovid <- dat_claims %>%
  filter(dataset == "mm-covid") %>%
  mutate(claim_text = str_to_lower(claim)) %>%
  unnest_tokens(word, claim) %>%
  anti_join(stop_words)

word_freq <- mmcovid  %>%
  count(veracity, word) %>%
  group_by(word) %>%
  mutate(freq = n / sum(n))

word_freq <- word_freq %>%
  filter(n >= 10)

word_freq_clean <- word_freq %>%
  filter(!is.na(word), word != "di", word != "de", word != "da")

# Top 40 words
top_40_words <- word_freq_clean %>%
  group_by(word) %>%
  summarize(total_n = sum(n)) %>% 
  arrange(desc(total_n)) %>%
  slice_head(n = 40)

word_freq_top_40 <- word_freq_clean %>%
  filter(word %in% top_40_words$word)

# Mean frequency
mean_freq <- word_freq_top_40 %>%
  pull(freq) %>% 
  mean(na.rm = TRUE)

mean_freq

### Graph

mmcovid_graph <- ggplot(word_freq_top_40, aes(x = freq, 
                             y = veracity, 
                             label = word)) +
  geom_point(size = 1) +
  geom_text_repel(color = "black",
                  size = 3, 
                  nudge_x = 0.03,
                  nudge_y = 0.1,
                  box.padding = 0.4,
                  max.overlaps = 50,
                  segment.size = 0.3,
                  segment.color = "grey") +
  labs(x = "Word Frequency (%)",
       y = "Veracity") +
  theme_minimal() +
  theme(plot.title = element_text(hjust = 0.5, size = 14),
        plot.subtitle = element_text(hjust = 0.5, size = 12), 
        panel.grid.minor.x = element_blank(),
        panel.grid.minor.y = element_blank(),
        axis.text = element_text(size = 6),
        axis.title = element_text(size = 10),
        axis.text.x = element_text(size = 8),
        axis.text.y = element_text(size = 8),
        axis.line = element_line(color = "gray47"), 
        axis.ticks = element_line(color = "gray47")) +
  scale_x_continuous(limits = c(0.59, 1.1), 
                     breaks = seq(0.6, 1, by = 0.1),  
                     expand = expansion(add = c(0, 0.1))) +  
  scale_y_reverse(limits = c(2.05, 0.5),
                  breaks = c(1, 2),
                  labels = c("True", "False"))
mmcovid_graph

ggsave(mmcovid_graph,
       filename = "mmcovid_graph.png",
       dpi = 320,
       bg = "white",
       units = "cm",
       height = 12,
       width = 18)


## TWITTER16

# Cleaning
twitter16_data <- dat_claims %>%
  filter(dataset == "twitter16") %>%
  mutate(claim_text = str_to_lower(claim)) %>%
  unnest_tokens(word, claim) %>%
  anti_join(stop_words)

word_freq <- twitter16_data %>%
  count(veracity, word) %>%
  group_by(word) %>%
  mutate(freq = n / sum(n))

word_freq <- word_freq %>%
  filter(n >= 10)

word_freq_clean <- word_freq %>%
  filter(!is.na(word), word != "url")

# 40 top frequent words
top_40_words <- word_freq_clean %>%
  group_by(word) %>%
  summarize(total_n = sum(n)) %>% 
  arrange(desc(total_n)) %>%
  slice_head(n = 40)

word_freq_top_40 <- word_freq_clean %>%
  filter(word %in% top_40_words$word)

# Mean frequency
mean_freq <- word_freq_top_40 %>%
  pull(freq) %>% 
  mean(na.rm = TRUE)

mean_freq

# Graph creation
twitter16_graph <- ggplot(word_freq_top_40, aes(x = freq, 
                                               y = veracity, 
                                               label = word)) +
  geom_point(size = 1) +
  geom_text_repel(color = "black",
                  size = 3, 
                  nudge_x = 0.03,
                  nudge_y = 0.1,
                  box.padding = 0.4,
                  max.overlaps = 50,
                  segment.size = 0.3,
                  segment.color = "grey") +
  labs(
       x = "Word Frequency (%)",
       y = "Veracity") +
  theme_minimal() +
  theme(plot.title = element_text(hjust = 0.5, size = 14),
        plot.subtitle = element_text(hjust = 0.5, size = 12), 
        panel.grid.minor.x = element_blank(),
        panel.grid.minor.y = element_blank(),
        axis.text = element_text(size = 6),
        axis.title = element_text(size = 10),
        axis.text.x = element_text(size = 8),
        axis.text.y = element_text(size = 8),
        axis.line = element_line(color = "gray47"), 
        axis.ticks = element_line(color = "gray47")) +
  scale_x_continuous(limits = c(0.59, 1.1), 
                     breaks = seq(0.6, 1, by = 0.1),  
                     expand = expansion(add = c(0, 0.1))) +  
  scale_y_reverse(limits = c(2.05, 0.5),
                  breaks = c(1, 2),
                  labels = c("True", "False"))

twitter16_graph

# Save
ggsave(twitter16_graph,
       filename = "twitter16_graph.png",
       dpi = 320,
       bg = "white",
       units = "cm",
       height = 12,
       width = 18)


## TRUTHSEEKER2023

# Cleaning
truthseeker2023_data <- dat_claims %>%
  filter(dataset == "truthseeker2023") %>%
  mutate(claim_text = str_to_lower(claim)) %>%
  unnest_tokens(word, claim) %>%
  anti_join(stop_words)

word_freq <- truthseeker2023_data %>%
  count(veracity, word) %>%
  group_by(word) %>%
  mutate(freq = n / sum(n))

word_freq <- word_freq %>%
  filter(n >= 2000)

word_freq_clean <- word_freq %>%
  filter(!is.na(word))

# Top 40 words

top_40_words <- word_freq_clean %>%
  group_by(word) %>%
  summarize(total_n = sum(n)) %>% 
  arrange(desc(total_n)) %>%
  slice_head(n = 40)

word_freq_top_40 <- word_freq_clean %>%
  filter(word %in% top_40_words$word)

# Mean frequency
mean_freq <- word_freq_top_40 %>%
  pull(freq) %>% 
  mean(na.rm = TRUE)

mean_freq

# Graph
truthseeker2023_graph <- ggplot(word_freq_top_40, aes(x = freq, 
                                                     y = veracity, 
                                                     label = word)) +
  geom_point(size = 1) +
  geom_text_repel(color = "black",
                  size = 3, 
                  nudge_x = 0.04,
                  nudge_y = 0.1,
                  box.padding = 0.4,
                  max.overlaps = 50,
                  segment.size = 0.3,
                  segment.color = "grey") +
  labs(x = "Word Frequency (%)",
       y = "Veracity") +
  theme_minimal() +
  theme(plot.title = element_text(hjust = 0.5, size = 14),
        plot.subtitle = element_text(hjust = 0.5, size = 12), 
        panel.grid.minor.x = element_blank(),
        panel.grid.minor.y = element_blank(),
        axis.text = element_text(size = 6),
        axis.title = element_text(size = 10),
        axis.text.x = element_text(size = 8),
        axis.text.y = element_text(size = 8),
        axis.line = element_line(color = "gray47"), 
        axis.ticks = element_line(color = "gray47")) +
  scale_x_continuous(limits = c(0.59, 1.1), 
                     breaks = seq(0.6, 1, by = 0.1),  
                     expand = expansion(add = c(0, 0.1))) +
  scale_y_reverse(limits = c(3.05, 0.5),
                  breaks = c(1, 2),
                  labels = c("True", "False"))

truthseeker2023_graph

ggsave(truthseeker2023_graph,
       filename = "truthseeker2023_graph.png",
       dpi = 320,
       bg = "white",
       units = "cm",
       height = 12,
       width = 18)
```