Hybrid Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a1931a6",
   "metadata": {},
   "source": [
    "# Hybrid Model\n",
    "\n",
    "The Hybrid Model used is a combination of Convolutional Neural Network (CNN) and Bidirectional Long-Short-Term-Memory Network (BiLSTM) to form a CNN-BiLSTM model by stacking the layers together. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd264244",
   "metadata": {},
   "source": [
    "## 1. Import Libraries\n",
    "Libraries needed for are imported in this step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8013fe0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import gensim\n",
    "import contractions\n",
    "import string\n",
    "import nltk\n",
    "import keras\n",
    "from cleantext import clean\n",
    "from gensim.models import word2vec\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.tokenize import TreebankWordDetokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras_preprocessing.sequence import pad_sequences\n",
    "from keras import regularizers\n",
    "from keras.layers import Embedding\n",
    "from keras.models import Sequential\n",
    "from keras import layers\n",
    "from keras import regularizers\n",
    "from keras import backend as K\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from sklearn.metrics import classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60a3c659",
   "metadata": {},
   "source": [
    "## 2. Read Training Dataset\n",
    "The training dataset used is the dataset downloaded from Kaggle that contains around 27k rows of Tweets with their sentiments labeled."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2aa670b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial: \n",
      "       textID                                               text  \\\n",
      "0  cb774db0d1                I`d have responded, if I were going   \n",
      "1  549e992a42      Sooo SAD I will miss you here in San Diego!!!   \n",
      "2  088c60f138                          my boss is bullying me...   \n",
      "3  9642c003ef                     what interview! leave me alone   \n",
      "4  358bd9e861   Sons of ****, why couldn`t they put them on t...   \n",
      "\n",
      "                         selected_text sentiment Time of Tweet Age of User  \\\n",
      "0  I`d have responded, if I were going   neutral       morning        0-20   \n",
      "1                             Sooo SAD  negative          noon       21-30   \n",
      "2                          bullying me  negative         night       31-45   \n",
      "3                       leave me alone  negative       morning       46-60   \n",
      "4                        Sons of ****,  negative          noon       60-70   \n",
      "\n",
      "       Country  Population -2020  Land Area (Km²)  Density (P/Km²)  \n",
      "0  Afghanistan          38928346         652860.0               60  \n",
      "1      Albania           2877797          27400.0              105  \n",
      "2      Algeria          43851044        2381740.0               18  \n",
      "3      Andorra             77265            470.0              164  \n",
      "4       Angola          32866272        1246700.0               26  \n"
     ]
    }
   ],
   "source": [
    "# read training dataset\n",
    "train = pd.read_csv(\"train.csv\", encoding = \"unicode_escape\")\n",
    "print(\"Initial: \")\n",
    "print(train.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0275f6ac",
   "metadata": {},
   "source": [
    "## 3. Drop Rows with Missing Values\n",
    "Rows that contain missing values are dropped to clean the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eaba7c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop rows with null values\n",
    "train = train.dropna(axis = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0aa7b2ab",
   "metadata": {},
   "source": [
    "## 4. Preprocess Texts\n",
    "The steps used to preprocess texts are as follows: <br>\n",
    "a. Remove Links <br>\n",
    "b. Remove Remove Emails <br>\n",
    "c. Remove New Line Characters <br>\n",
    "d. Remove Numbers <br>\n",
    "e. Remove Emojis <br>\n",
    "f. Remove Punctuations and Accents <br>\n",
    "g. Remove Irrelevant Words (Length shorter or equal to 2 OR length greater or equal to 15) <br>\n",
    "h. Remove Stopwords <br>\n",
    "i. Tokenization <br>\n",
    "j. Part-of-Speech (POS) Tagging <br>\n",
    "k. Lemmatization <br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "40d638cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['respond go', 'sooo sad miss san diego', 'bos bully', 'interview leave alone', 'son put release already buy']\n"
     ]
    }
   ],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "def get_wordnet_pos(treebank_pos):\n",
    "    if treebank_pos.startswith('J'):\n",
    "        return wordnet.ADJ\n",
    "    elif treebank_pos.startswith('V'):\n",
    "        return wordnet.VERB\n",
    "    elif treebank_pos.startswith('N'):\n",
    "        return wordnet.NOUN\n",
    "    elif treebank_pos.startswith('R'):\n",
    "        return wordnet.ADV\n",
    "    else:\n",
    "        return wordnet.NOUN\n",
    "\n",
    "def depure_data(data):\n",
    "    # Remove URL\n",
    "    url_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "    data = url_pattern.sub(r'', data)\n",
    "\n",
    "    # Remove Emails\n",
    "    data = re.sub('\\S*@\\S*\\s?', '', data)\n",
    "\n",
    "    # Remove new line characters\n",
    "    data = re.sub('\\s+', ' ', data)\n",
    "    \n",
    "    # Remove distracting single quotes\n",
    "    data = re.sub(\"\\'\", \"\", data)\n",
    "        \n",
    "    return data\n",
    "\n",
    "def sent_to_words(sentences):\n",
    "    for sentence in sentences:\n",
    "        # 1. Converts to lowercase\n",
    "        # 2. Removes accents and punctuations\n",
    "        # 3. Removes words shorter (<=) or longer (>=) than minimum length of 2 and maximum length of 15\n",
    "        # 4. Removes numerical characters\n",
    "        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))\n",
    "        \n",
    "def detokenize(text):\n",
    "    return TreebankWordDetokenizer().detokenize(text)\n",
    "\n",
    "def stopwords_lemmatize(data):\n",
    "    # Remove stopwords\n",
    "    data = \" \".join([word for word in str(data).split() if word not in stop_words])\n",
    "    \n",
    "    # Apply tokenization\n",
    "    data = nltk.word_tokenize(data)\n",
    "    \n",
    "    # Perform POS tagging\n",
    "    data = nltk.pos_tag(data)\n",
    "    \n",
    "    # Perform lemmatization\n",
    "    lemmatized_tokens = []\n",
    "\n",
    "    for i in range(len(data)):\n",
    "        word, pos = data[i]\n",
    "        \n",
    "        lemmatized_tokens.append(nltk.WordNetLemmatizer().lemmatize(word, get_wordnet_pos(pos)))\n",
    "    \n",
    "    data = \" \".join(lemmatized_tokens)\n",
    "    \n",
    "    return data\n",
    "\n",
    "temp = []\n",
    "# Convert the values into a list\n",
    "data_to_list = train['text'].values.tolist()\n",
    "for i in range(len(data_to_list)):\n",
    "    temp.append(depure_data(data_to_list[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20d275b9",
   "metadata": {},
   "source": [
    "## 5. Label Encoding\n",
    "The label is encoded into 1 for positive, 0 for neutral and -1 for negative before it is one-hot encoded into the following structure: <br>\n",
    "[neutral, positive, negative]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b23e1af9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "train['sentiment'] = train['sentiment'].map({\"positive\": 1, \"neutral\": 0, \"negative\": -1})\n",
    "y = train['sentiment'].to_numpy()\n",
    "y = keras.utils.to_categorical(y, 3)\n",
    "print(y[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5587ab7e",
   "metadata": {},
   "source": [
    "## 6. Feature Extraction\n",
    "The feature vector or raw word vector is extracted by converting the texts into sequence of numbers. <br>\n",
    "The maximum number of words that will be stored in the tokenizer is 5000 and the maximum length of the vector is 200, texts that are shorter than the maximum length will be padded with 0s. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d9b31c05",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[   0    0    0 ...    0 1208    2]\n",
      " [   0    0    0 ...   19 1209 1898]\n",
      " [   0    0    0 ...    0 1162 3767]\n",
      " ...\n",
      " [   0    0    0 ...  434  668 2348]\n",
      " [   0    0    0 ...    0    0  523]\n",
      " [   0    0    0 ...  460  143  312]]\n"
     ]
    }
   ],
   "source": [
    "max_words = 5000\n",
    "max_len = 200\n",
    "\n",
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "tokenizer.fit_on_texts(data)\n",
    "sequences = tokenizer.texts_to_sequences(data)\n",
    "tweets = pad_sequences(sequences, maxlen=max_len)\n",
    "print(tweets)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50b7e0e7",
   "metadata": {},
   "source": [
    "## 7. Train Validation Split\n",
    "The training dataset is split by 80% and 20% into train and validation set. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e62e431f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(tweets, y, test_size = 0.2, random_state = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b9fc40a",
   "metadata": {},
   "source": [
    "## 8. Model Training\n",
    "The models in this file are the two individual models, CNN, BiLSTM and the hybrid model, CNN-BiLSTM. After each model is trained, the model is saved to a .h5 file so that the model can be loaded for prediction without training again. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b49ba9f0",
   "metadata": {},
   "source": [
    "### a. BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "95d78f37",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_3 (Embedding)     (None, 200, 40)           200000    \n",
      "                                                                 \n",
      " bidirectional_3 (Bidirectio  (None, 40)               9760      \n",
      " nal)                                                            \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 3)                 123       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 209,883\n",
      "Trainable params: 209,883\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.\n",
      "Epoch 1/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.9992 - accuracy: 0.4944\n",
      "Epoch 1: val_accuracy improved from -inf to 0.64793, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 28s 38ms/step - loss: 0.9991 - accuracy: 0.4944 - val_loss: 0.8292 - val_accuracy: 0.6479\n",
      "Epoch 2/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.8465 - accuracy: 0.6169\n",
      "Epoch 2: val_accuracy improved from 0.64793 to 0.68395, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 31s 45ms/step - loss: 0.8465 - accuracy: 0.6169 - val_loss: 0.7543 - val_accuracy: 0.6840\n",
      "Epoch 3/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.7891 - accuracy: 0.6608\n",
      "Epoch 3: val_accuracy improved from 0.68395 to 0.69414, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 30s 44ms/step - loss: 0.7891 - accuracy: 0.6608 - val_loss: 0.7328 - val_accuracy: 0.6941\n",
      "Epoch 4/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.7634 - accuracy: 0.6748\n",
      "Epoch 4: val_accuracy improved from 0.69414 to 0.70597, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 30s 44ms/step - loss: 0.7634 - accuracy: 0.6748 - val_loss: 0.7128 - val_accuracy: 0.7060\n",
      "Epoch 5/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.7413 - accuracy: 0.6836\n",
      "Epoch 5: val_accuracy improved from 0.70597 to 0.71143, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 31s 45ms/step - loss: 0.7413 - accuracy: 0.6836 - val_loss: 0.7039 - val_accuracy: 0.7114\n",
      "Epoch 6/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.7270 - accuracy: 0.6941\n",
      "Epoch 6: val_accuracy improved from 0.71143 to 0.71434, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 32s 46ms/step - loss: 0.7270 - accuracy: 0.6942 - val_loss: 0.6989 - val_accuracy: 0.7143\n",
      "Epoch 7/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.7135 - accuracy: 0.7036\n",
      "Epoch 7: val_accuracy improved from 0.71434 to 0.71689, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 32s 46ms/step - loss: 0.7134 - accuracy: 0.7037 - val_loss: 0.6930 - val_accuracy: 0.7169\n",
      "Epoch 8/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.7050 - accuracy: 0.7065\n",
      "Epoch 8: val_accuracy improved from 0.71689 to 0.71852, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 34s 49ms/step - loss: 0.7050 - accuracy: 0.7065 - val_loss: 0.6882 - val_accuracy: 0.7185\n",
      "Epoch 9/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.6915 - accuracy: 0.7157\n",
      "Epoch 9: val_accuracy improved from 0.71852 to 0.71980, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 34s 50ms/step - loss: 0.6915 - accuracy: 0.7157 - val_loss: 0.6912 - val_accuracy: 0.7198\n",
      "Epoch 10/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.6844 - accuracy: 0.7157\n",
      "Epoch 10: val_accuracy improved from 0.71980 to 0.72325, saving model to best_model0.hdf5\n",
      "687/687 [==============================] - 35s 50ms/step - loss: 0.6844 - accuracy: 0.7157 - val_loss: 0.6920 - val_accuracy: 0.7233\n"
     ]
    }
   ],
   "source": [
    "model0 = Sequential()\n",
    "model0.add(layers.Embedding(max_words, 40, input_length=max_len))\n",
    "model0.add(layers.Bidirectional(layers.LSTM(20,dropout=0.9)))\n",
    "model0.add(layers.Dense(3, activation='softmax'))\n",
    "model0.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model0.summary()\n",
    "\n",
    "checkpoint0 = ModelCheckpoint(\"best_model0.hdf5\", monitor='val_accuracy', verbose=1,save_best_only=True, mode='auto', period=1,save_weights_only=False)\n",
    "history = model0.fit(X_train, y_train, epochs=10, validation_data=(X_test, y_test),callbacks=[checkpoint0])\n",
    "model0.save('BiLSTM.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db4bf628",
   "metadata": {},
   "source": [
    "### b. CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1d017d17",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_6 (Embedding)     (None, 200, 40)           200000    \n",
      "                                                                 \n",
      " conv1d (Conv1D)             (None, 198, 32)           3872      \n",
      "                                                                 \n",
      " max_pooling1d (MaxPooling1D  (None, 99, 32)           0         \n",
      " )                                                               \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 99, 32)            0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 3168)              0         \n",
      "                                                                 \n",
      " dense_6 (Dense)             (None, 10)                31690     \n",
      "                                                                 \n",
      " dense_7 (Dense)             (None, 3)                 33        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 235,595\n",
      "Trainable params: 235,595\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/8\n",
      "344/344 [==============================] - 4s 11ms/step - loss: 1.1674 - accuracy: 0.4012 - val_loss: 1.0882 - val_accuracy: 0.4139\n",
      "Epoch 2/8\n",
      "344/344 [==============================] - 3s 10ms/step - loss: 1.0878 - accuracy: 0.4022 - val_loss: 1.0561 - val_accuracy: 0.4139\n",
      "Epoch 3/8\n",
      "344/344 [==============================] - 3s 10ms/step - loss: 1.0099 - accuracy: 0.5169 - val_loss: 0.9239 - val_accuracy: 0.5730\n",
      "Epoch 4/8\n",
      "344/344 [==============================] - 4s 11ms/step - loss: 0.9572 - accuracy: 0.5584 - val_loss: 0.8749 - val_accuracy: 0.6248\n",
      "Epoch 5/8\n",
      "344/344 [==============================] - 5s 14ms/step - loss: 0.9187 - accuracy: 0.6001 - val_loss: 0.8418 - val_accuracy: 0.6578\n",
      "Epoch 6/8\n",
      "344/344 [==============================] - 5s 14ms/step - loss: 0.9042 - accuracy: 0.6091 - val_loss: 0.8351 - val_accuracy: 0.6907\n",
      "Epoch 7/8\n",
      "344/344 [==============================] - 5s 15ms/step - loss: 0.8929 - accuracy: 0.6165 - val_loss: 0.8240 - val_accuracy: 0.6810\n",
      "Epoch 8/8\n",
      "344/344 [==============================] - 5s 13ms/step - loss: 0.8793 - accuracy: 0.6263 - val_loss: 0.8231 - val_accuracy: 0.6745\n"
     ]
    }
   ],
   "source": [
    "model1 = Sequential()\n",
    "model1.add(layers.Embedding(max_words, 40, input_length=max_len))\n",
    "model1.add(layers.Conv1D(filters=32, kernel_size=3, activation='relu', kernel_regularizer=regularizers.l2(l=0.01)))\n",
    "model1.add(layers.MaxPooling1D(pool_size=2))\n",
    "model1.add(layers.Dropout(0.9))\n",
    "model1.add(layers.Flatten())\n",
    "model1.add(layers.Dense(10, activation='relu', kernel_regularizer=regularizers.l2(l=0.01)))\n",
    "model1.add(layers.Dense(3, activation='softmax'))\n",
    "model1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model1.summary()\n",
    "\n",
    "history = model1.fit(X_train, y_train, epochs=8, batch_size=64, validation_data=(X_test, y_test))\n",
    "model1.save('CNN.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "926d43f2",
   "metadata": {},
   "source": [
    "### c. CNN-BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e642c38d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " embedding_2 (Embedding)     (None, 200, 40)           200000    \n",
      "                                                                 \n",
      " conv1d_2 (Conv1D)           (None, 198, 32)           3872      \n",
      "                                                                 \n",
      " max_pooling1d_2 (MaxPooling  (None, 99, 32)           0         \n",
      " 1D)                                                             \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 99, 32)            0         \n",
      "                                                                 \n",
      " bidirectional_2 (Bidirectio  (None, 60)               15120     \n",
      " nal)                                                            \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 10)                610       \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 3)                 33        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 219,635\n",
      "Trainable params: 219,635\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.\n",
      "Epoch 1/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 1.1393 - accuracy: 0.4019\n",
      "Epoch 1: val_accuracy improved from -inf to 0.41394, saving model to best_model2.hdf5\n",
      "687/687 [==============================] - 26s 35ms/step - loss: 1.1392 - accuracy: 0.4018 - val_loss: 1.0863 - val_accuracy: 0.4139\n",
      "Epoch 2/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 1.0883 - accuracy: 0.4020\n",
      "Epoch 2: val_accuracy did not improve from 0.41394\n",
      "687/687 [==============================] - 25s 37ms/step - loss: 1.0883 - accuracy: 0.4022 - val_loss: 1.0844 - val_accuracy: 0.4139\n",
      "Epoch 3/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 1.0879 - accuracy: 0.4023\n",
      "Epoch 3: val_accuracy did not improve from 0.41394\n",
      "687/687 [==============================] - 26s 37ms/step - loss: 1.0880 - accuracy: 0.4022 - val_loss: 1.0845 - val_accuracy: 0.4139\n",
      "Epoch 4/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 1.0104 - accuracy: 0.4870\n",
      "Epoch 4: val_accuracy improved from 0.41394 to 0.58406, saving model to best_model2.hdf5\n",
      "687/687 [==============================] - 25s 36ms/step - loss: 1.0101 - accuracy: 0.4872 - val_loss: 0.8982 - val_accuracy: 0.5841\n",
      "Epoch 5/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.8763 - accuracy: 0.6235\n",
      "Epoch 5: val_accuracy improved from 0.58406 to 0.68468, saving model to best_model2.hdf5\n",
      "687/687 [==============================] - 24s 35ms/step - loss: 0.8763 - accuracy: 0.6235 - val_loss: 0.8147 - val_accuracy: 0.6847\n",
      "Epoch 6/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.8181 - accuracy: 0.6674\n",
      "Epoch 6: val_accuracy improved from 0.68468 to 0.69796, saving model to best_model2.hdf5\n",
      "687/687 [==============================] - 23s 34ms/step - loss: 0.8181 - accuracy: 0.6674 - val_loss: 0.7797 - val_accuracy: 0.6980\n",
      "Epoch 7/10\n",
      "685/687 [============================>.] - ETA: 0s - loss: 0.7834 - accuracy: 0.6882\n",
      "Epoch 7: val_accuracy did not improve from 0.69796\n",
      "687/687 [==============================] - 25s 37ms/step - loss: 0.7833 - accuracy: 0.6883 - val_loss: 0.7796 - val_accuracy: 0.6969\n",
      "Epoch 8/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.7642 - accuracy: 0.6998\n",
      "Epoch 8: val_accuracy did not improve from 0.69796\n",
      "687/687 [==============================] - 24s 34ms/step - loss: 0.7641 - accuracy: 0.6997 - val_loss: 0.7748 - val_accuracy: 0.6980\n",
      "Epoch 9/10\n",
      "686/687 [============================>.] - ETA: 0s - loss: 0.7537 - accuracy: 0.7039\n",
      "Epoch 9: val_accuracy did not improve from 0.69796\n",
      "687/687 [==============================] - 24s 34ms/step - loss: 0.7538 - accuracy: 0.7038 - val_loss: 0.7694 - val_accuracy: 0.6961\n",
      "Epoch 10/10\n",
      "687/687 [==============================] - ETA: 0s - loss: 0.7429 - accuracy: 0.7121\n",
      "Epoch 10: val_accuracy improved from 0.69796 to 0.70670, saving model to best_model2.hdf5\n",
      "687/687 [==============================] - 24s 34ms/step - loss: 0.7429 - accuracy: 0.7121 - val_loss: 0.7590 - val_accuracy: 0.7067\n"
     ]
    }
   ],
   "source": [
    "model2 = Sequential()\n",
    "model2.add(layers.Embedding(max_words, 40, input_length=max_len)) \n",
    "model2.add(layers.Conv1D(filters=32, kernel_size=3, activation='relu', kernel_regularizer=regularizers.l2(l=0.01)))\n",
    "model2.add(layers.MaxPooling1D(pool_size=2))\n",
    "model2.add(layers.Dropout(0.6))\n",
    "model2.add(layers.Bidirectional(layers.LSTM(30,dropout=0.7)))\n",
    "model2.add(layers.Dense(10, activation='relu', kernel_regularizer=regularizers.l2(l=0.01)))\n",
    "model2.add(layers.Dense(3, activation='softmax'))\n",
    "model2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "model2.summary()\n",
    "\n",
    "checkpoint2 = ModelCheckpoint(\"best_model2.hdf5\", monitor='val_accuracy', verbose=1,save_best_only=True, mode='auto', period=1,save_weights_only=False)\n",
    "history = model2.fit(X_train, y_train, epochs=10, validation_data=(X_test, y_test),callbacks=[checkpoint2])\n",
    "model2.save('hybrid.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4683462b",
   "metadata": {},
   "source": [
    "## 9. Model Evaluation\n",
    "Each of the models are evaluated using accuracy, precision, recall and F1-score which are obtained from the classification report. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4152b2e",
   "metadata": {},
   "source": [
    "### a. BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a41a0dcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " BiLSTM MODEL \n",
      "==============\n",
      " TRAIN ACCURACY \n",
      "================\n",
      "687/687 [==============================] - 9s 11ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.84      0.65      0.73      6258\n",
      "     neutral       0.67      0.86      0.76      8842\n",
      "    positive       0.86      0.73      0.79      6884\n",
      "\n",
      "    accuracy                           0.76     21984\n",
      "   macro avg       0.79      0.75      0.76     21984\n",
      "weighted avg       0.78      0.76      0.76     21984\n",
      "\n",
      " VALIDATION ACCURACY \n",
      "=====================\n",
      "172/172 [==============================] - 3s 11ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.79      0.59      0.67      1523\n",
      "     neutral       0.65      0.83      0.73      2275\n",
      "    positive       0.82      0.70      0.75      1698\n",
      "\n",
      "    accuracy                           0.72      5496\n",
      "   macro avg       0.75      0.70      0.72      5496\n",
      "weighted avg       0.74      0.72      0.72      5496\n",
      "\n",
      " REVIEW TEST ACCURACY \n",
      "======================\n",
      "['anyone iphone pro iphone', 'remember first time buy smart phone iphone show grandfather proud could first question able make rich try prove right time crypto', 'iphone pro max take use new iphone notice interesting change use also many similarity iphone pro max user new feature change come across quite subtle', 'start save next iphone whatever pro', 'mf iphone fuck update everything become slow even pay apps rather switch anaroid stupid io']\n",
      "10/10 [==============================] - 0s 12ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.71      0.32      0.44       126\n",
      "     neutral       0.29      0.82      0.43        77\n",
      "    positive       0.79      0.29      0.42       107\n",
      "\n",
      "    accuracy                           0.43       310\n",
      "   macro avg       0.60      0.48      0.43       310\n",
      "weighted avg       0.64      0.43      0.43       310\n",
      "\n",
      " COVID TEST ACCURACY \n",
      "=====================\n",
      "['notice one wear mask everybody seem pretty good shape joe biden covid covid covidisnotover longcovid', 'dont positive time perfectly okay feel sad angry annoy frustrate scared anxious feeling doesnt make negative person make human lori deschene mentalhealth health anxiety depression covid support nft', 'legal happy million spend past year mourn death love one due covid last thing want need fuckimg sadness especially mandatory sadness rather watch spongebob monday notmyking', 'ccbq invite wellness wednesday september downtown brooklyn see fly information bethesolution resource service nyc covid coronavirus corona community love compassion socialgood joy philanthropy gratitude', 'covid vaccine booster omicron dont time clinical trial dont know degree vaccine wan yearly booster require happy natural immunity work adverse effect amp yearly booster cbc canada']\n",
      "10/10 [==============================] - 0s 11ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.62      0.69      0.65       100\n",
      "     neutral       0.51      0.57      0.54       102\n",
      "    positive       0.81      0.59      0.68        88\n",
      "\n",
      "    accuracy                           0.62       290\n",
      "   macro avg       0.65      0.62      0.62       290\n",
      "weighted avg       0.64      0.62      0.62       290\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model0 = keras.models.load_model('BiLSTM.h5')\n",
    "\n",
    "sentiment = ['neutral', 'positive', 'negative']\n",
    "\n",
    "print(\" BiLSTM MODEL \")\n",
    "print(\"==============\")\n",
    "\n",
    "print(\" TRAIN ACCURACY \")\n",
    "print(\"================\")\n",
    "y_predTrain = model0.predict(X_train)\n",
    "y_train_transform = []\n",
    "for y in y_train:\n",
    "    y_train_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_train = []\n",
    "for y in y_predTrain:\n",
    "    predicted_train.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_train_transform, predicted_train))\n",
    "\n",
    "\n",
    "print(\" VALIDATION ACCURACY \")\n",
    "print(\"=====================\")\n",
    "y_predTest = model0.predict(X_test)\n",
    "y_test_transform = []\n",
    "for y in y_test:\n",
    "    y_test_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_test = []\n",
    "for y in y_predTest:\n",
    "    predicted_test.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_test_transform, predicted_test))\n",
    "\n",
    "\n",
    "print(\" REVIEW TEST ACCURACY \")\n",
    "print(\"======================\")\n",
    "reviewTest = pd.read_csv(\"iPhoneTest.csv\")\n",
    "X_reviewTest = reviewTest['text']\n",
    "y_reviewTest = reviewTest['sentiment'].values.tolist()\n",
    "\n",
    "X_reviewTest = X_reviewTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_reviewTest)):\n",
    "    temp.append(depure_data(X_reviewTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_reviewTest = tokenizer.texts_to_sequences(data)\n",
    "X_reviewTest = pad_sequences(X_reviewTest, maxlen=max_len)\n",
    "\n",
    "y_predictReview = model0.predict(X_reviewTest)\n",
    "predicted_sentiment_review = []\n",
    "for y in y_predictReview:\n",
    "    predicted_sentiment_review.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "print(classification_report(y_reviewTest, predicted_sentiment_review))\n",
    "\n",
    "print(\" COVID TEST ACCURACY \")\n",
    "print(\"=====================\")\n",
    "covidTest = pd.read_csv(\"covidTest.csv\")\n",
    "X_covidTest = covidTest['text']\n",
    "y_covidTest = covidTest['sentiment'].values.tolist()\n",
    "\n",
    "X_covidTest = X_covidTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_covidTest)):\n",
    "    temp.append(depure_data(X_covidTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_covidTest = tokenizer.texts_to_sequences(data)\n",
    "X_covidTest = pad_sequences(X_covidTest, maxlen=max_len)\n",
    "\n",
    "y_predictCovid = model0.predict(X_covidTest)\n",
    "predicted_sentiment_covid = []\n",
    "for y in y_predictCovid:\n",
    "    predicted_sentiment_covid.append(sentiment[np.around(y, decimals=0).argmax()]) \n",
    "print(classification_report(y_covidTest, predicted_sentiment_covid))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3d92bd1",
   "metadata": {},
   "source": [
    "### b. CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "88f8e90a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " CNN MODEL \n",
      "===========\n",
      " TRAIN ACCURACY \n",
      "================\n",
      "687/687 [==============================] - 2s 3ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.87      0.41      0.56      6258\n",
      "     neutral       0.60      0.88      0.71      8842\n",
      "    positive       0.85      0.74      0.79      6884\n",
      "\n",
      "    accuracy                           0.70     21984\n",
      "   macro avg       0.77      0.68      0.69     21984\n",
      "weighted avg       0.75      0.70      0.69     21984\n",
      "\n",
      " VALIDATION ACCURACY \n",
      "=====================\n",
      "172/172 [==============================] - 1s 3ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.82      0.35      0.49      1523\n",
      "     neutral       0.57      0.84      0.68      2275\n",
      "    positive       0.78      0.68      0.73      1698\n",
      "\n",
      "    accuracy                           0.66      5496\n",
      "   macro avg       0.72      0.63      0.63      5496\n",
      "weighted avg       0.70      0.66      0.64      5496\n",
      "\n",
      " REVIEW TEST ACCURACY \n",
      "======================\n",
      "['anyone iphone pro iphone', 'remember first time buy smart phone iphone show grandfather proud could first question able make rich try prove right time crypto', 'iphone pro max take use new iphone notice interesting change use also many similarity iphone pro max user new feature change come across quite subtle', 'start save next iphone whatever pro', 'mf iphone fuck update everything become slow even pay apps rather switch anaroid stupid io']\n",
      "10/10 [==============================] - 0s 3ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.74      0.16      0.26       126\n",
      "     neutral       0.28      0.88      0.43        77\n",
      "    positive       0.81      0.33      0.47       107\n",
      "\n",
      "    accuracy                           0.40       310\n",
      "   macro avg       0.61      0.46      0.39       310\n",
      "weighted avg       0.65      0.40      0.37       310\n",
      "\n",
      " COVID TEST ACCURACY \n",
      "=====================\n",
      "['notice one wear mask everybody seem pretty good shape joe biden covid covid covidisnotover longcovid', 'dont positive time perfectly okay feel sad angry annoy frustrate scared anxious feeling doesnt make negative person make human lori deschene mentalhealth health anxiety depression covid support nft', 'legal happy million spend past year mourn death love one due covid last thing want need fuckimg sadness especially mandatory sadness rather watch spongebob monday notmyking', 'ccbq invite wellness wednesday september downtown brooklyn see fly information bethesolution resource service nyc covid coronavirus corona community love compassion socialgood joy philanthropy gratitude', 'covid vaccine booster omicron dont time clinical trial dont know degree vaccine wan yearly booster require happy natural immunity work adverse effect amp yearly booster cbc canada']\n",
      "10/10 [==============================] - 0s 3ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.71      0.49      0.58       100\n",
      "     neutral       0.49      0.74      0.59       102\n",
      "    positive       0.80      0.62      0.70        88\n",
      "\n",
      "    accuracy                           0.62       290\n",
      "   macro avg       0.67      0.62      0.62       290\n",
      "weighted avg       0.66      0.62      0.62       290\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model1 = keras.models.load_model('CNN.h5')\n",
    "\n",
    "sentiment = ['neutral', 'positive', 'negative']\n",
    "\n",
    "print(\" CNN MODEL \")\n",
    "print(\"===========\")\n",
    "\n",
    "print(\" TRAIN ACCURACY \")\n",
    "print(\"================\")\n",
    "y_predTrain = model1.predict(X_train)\n",
    "y_train_transform = []\n",
    "for y in y_train:\n",
    "    y_train_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_train = []\n",
    "for y in y_predTrain:\n",
    "    predicted_train.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_train_transform, predicted_train))\n",
    "\n",
    "\n",
    "print(\" VALIDATION ACCURACY \")\n",
    "print(\"=====================\")\n",
    "y_predTest = model1.predict(X_test)\n",
    "y_test_transform = []\n",
    "for y in y_test:\n",
    "    y_test_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_test = []\n",
    "for y in y_predTest:\n",
    "    predicted_test.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_test_transform, predicted_test))\n",
    "\n",
    "\n",
    "print(\" REVIEW TEST ACCURACY \")\n",
    "print(\"======================\")\n",
    "reviewTest = pd.read_csv(\"iPhoneTest.csv\")\n",
    "X_reviewTest = reviewTest['text']\n",
    "y_reviewTest = reviewTest['sentiment'].values.tolist()\n",
    "\n",
    "X_reviewTest = X_reviewTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_reviewTest)):\n",
    "    temp.append(depure_data(X_reviewTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_reviewTest = tokenizer.texts_to_sequences(data)\n",
    "X_reviewTest = pad_sequences(X_reviewTest, maxlen=max_len)\n",
    "\n",
    "y_predictReview = model1.predict(X_reviewTest)\n",
    "predicted_sentiment_review = []\n",
    "for y in y_predictReview:\n",
    "    predicted_sentiment_review.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "print(classification_report(y_reviewTest, predicted_sentiment_review))\n",
    "\n",
    "print(\" COVID TEST ACCURACY \")\n",
    "print(\"=====================\")\n",
    "covidTest = pd.read_csv(\"covidTest.csv\")\n",
    "X_covidTest = covidTest['text']\n",
    "y_covidTest = covidTest['sentiment'].values.tolist()\n",
    "\n",
    "X_covidTest = X_covidTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_covidTest)):\n",
    "    temp.append(depure_data(X_covidTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_covidTest = tokenizer.texts_to_sequences(data)\n",
    "X_covidTest = pad_sequences(X_covidTest, maxlen=max_len)\n",
    "\n",
    "y_predictCovid = model1.predict(X_covidTest)\n",
    "predicted_sentiment_covid = []\n",
    "for y in y_predictCovid:\n",
    "    predicted_sentiment_covid.append(sentiment[np.around(y, decimals=0).argmax()]) \n",
    "print(classification_report(y_covidTest, predicted_sentiment_covid))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d72ffd9",
   "metadata": {},
   "source": [
    "### c. CNN-BiLSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "08cf3594",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " HYBRID MODEL \n",
      "==============\n",
      " TRAIN ACCURACY \n",
      "================\n",
      "687/687 [==============================] - 17s 21ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.83      0.72      0.77      6258\n",
      "     neutral       0.73      0.83      0.78      8842\n",
      "    positive       0.87      0.82      0.84      6884\n",
      "\n",
      "    accuracy                           0.79     21984\n",
      "   macro avg       0.81      0.79      0.80     21984\n",
      "weighted avg       0.80      0.79      0.80     21984\n",
      "\n",
      " VALIDATION ACCURACY \n",
      "=====================\n",
      "172/172 [==============================] - 6s 22ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.73      0.62      0.67      1523\n",
      "     neutral       0.66      0.75      0.70      2275\n",
      "    positive       0.77      0.73      0.75      1698\n",
      "\n",
      "    accuracy                           0.71      5496\n",
      "   macro avg       0.72      0.70      0.71      5496\n",
      "weighted avg       0.71      0.71      0.71      5496\n",
      "\n",
      " REVIEW TEST ACCURACY \n",
      "======================\n",
      "['anyone iphone pro iphone', 'remember first time buy smart phone iphone show grandfather proud could first question able make rich try prove right time crypto', 'iphone pro max take use new iphone notice interesting change use also many similarity iphone pro max user new feature change come across quite subtle', 'start save next iphone whatever pro', 'mf iphone fuck update everything become slow even pay apps rather switch anaroid stupid io']\n",
      "10/10 [==============================] - 0s 20ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.62      0.40      0.49       126\n",
      "     neutral       0.30      0.65      0.41        77\n",
      "    positive       0.69      0.40      0.51       107\n",
      "\n",
      "    accuracy                           0.46       310\n",
      "   macro avg       0.54      0.48      0.47       310\n",
      "weighted avg       0.57      0.46      0.47       310\n",
      "\n",
      " COVID TEST ACCURACY \n",
      "=====================\n",
      "['notice one wear mask everybody seem pretty good shape joe biden covid covid covidisnotover longcovid', 'dont positive time perfectly okay feel sad angry annoy frustrate scared anxious feeling doesnt make negative person make human lori deschene mentalhealth health anxiety depression covid support nft', 'legal happy million spend past year mourn death love one due covid last thing want need fuckimg sadness especially mandatory sadness rather watch spongebob monday notmyking', 'ccbq invite wellness wednesday september downtown brooklyn see fly information bethesolution resource service nyc covid coronavirus corona community love compassion socialgood joy philanthropy gratitude', 'covid vaccine booster omicron dont time clinical trial dont know degree vaccine wan yearly booster require happy natural immunity work adverse effect amp yearly booster cbc canada']\n",
      "10/10 [==============================] - 0s 20ms/step\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "    negative       0.60      0.65      0.62       100\n",
      "     neutral       0.50      0.50      0.50       102\n",
      "    positive       0.72      0.66      0.69        88\n",
      "\n",
      "    accuracy                           0.60       290\n",
      "   macro avg       0.61      0.60      0.60       290\n",
      "weighted avg       0.60      0.60      0.60       290\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model2 = keras.models.load_model('hybrid.h5')\n",
    "\n",
    "sentiment = ['neutral', 'positive', 'negative']\n",
    "\n",
    "print(\" HYBRID MODEL \")\n",
    "print(\"==============\")\n",
    "\n",
    "print(\" TRAIN ACCURACY \")\n",
    "print(\"================\")\n",
    "y_predTrain = model2.predict(X_train)\n",
    "y_train_transform = []\n",
    "for y in y_train:\n",
    "    y_train_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_train = []\n",
    "for y in y_predTrain:\n",
    "    predicted_train.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_train_transform, predicted_train))\n",
    "\n",
    "\n",
    "print(\" VALIDATION ACCURACY \")\n",
    "print(\"=====================\")\n",
    "y_predTest = model2.predict(X_test)\n",
    "y_test_transform = []\n",
    "for y in y_test:\n",
    "    y_test_transform.append(sentiment[np.around(y, decimals = 0).argmax()])\n",
    "\n",
    "predicted_test = []\n",
    "for y in y_predTest:\n",
    "    predicted_test.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "    \n",
    "print(classification_report(y_test_transform, predicted_test))\n",
    "\n",
    "\n",
    "print(\" REVIEW TEST ACCURACY \")\n",
    "print(\"======================\")\n",
    "reviewTest = pd.read_csv(\"iPhoneTest.csv\")\n",
    "X_reviewTest = reviewTest['text']\n",
    "y_reviewTest = reviewTest['sentiment'].values.tolist()\n",
    "\n",
    "X_reviewTest = X_reviewTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_reviewTest)):\n",
    "    temp.append(depure_data(X_reviewTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_reviewTest = tokenizer.texts_to_sequences(data)\n",
    "X_reviewTest = pad_sequences(X_reviewTest, maxlen=max_len)\n",
    "\n",
    "y_predictReview = model2.predict(X_reviewTest)\n",
    "predicted_sentiment_review = []\n",
    "for y in y_predictReview:\n",
    "    predicted_sentiment_review.append(sentiment[np.around(y, decimals=0).argmax()])\n",
    "print(classification_report(y_reviewTest, predicted_sentiment_review))\n",
    "\n",
    "print(\" COVID TEST ACCURACY \")\n",
    "print(\"=====================\")\n",
    "covidTest = pd.read_csv(\"covidTest.csv\")\n",
    "X_covidTest = covidTest['text']\n",
    "y_covidTest = covidTest['sentiment'].values.tolist()\n",
    "\n",
    "X_covidTest = X_covidTest.values.tolist()\n",
    "temp = []\n",
    "for i in range(len(X_covidTest)):\n",
    "    temp.append(depure_data(X_covidTest[i]))\n",
    "data_words = list(sent_to_words(temp))\n",
    "detokenized = []\n",
    "for i in range(len(data_words)):\n",
    "    detokenized.append(detokenize(data_words[i]))\n",
    "data = []\n",
    "for i in range(len(detokenized)):\n",
    "    data.append(stopwords_lemmatize(detokenized[i]))\n",
    "print(data[:5])\n",
    "X_covidTest = tokenizer.texts_to_sequences(data)\n",
    "X_covidTest = pad_sequences(X_covidTest, maxlen=max_len)\n",
    "\n",
    "y_predictCovid = model2.predict(X_covidTest)\n",
    "predicted_sentiment_covid = []\n",
    "for y in y_predictCovid:\n",
    "    predicted_sentiment_covid.append(sentiment[np.around(y, decimals=0).argmax()]) \n",
    "print(classification_report(y_covidTest, predicted_sentiment_covid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0766874",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}