code/gijoong/02_cv_tfidf_compare.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "def kaggle_format(df):\n",
    "    df['label'][df['label'] == 'none'] = 0\n",
    "    df['label'][df['label'] == 'offensive'] = 1\n",
    "    df['label'][df['label'] == 'hate'] = 2\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# train 데이터 / 7893/ \n",
    "# 'none'/ 'offensive'/ 'hate' : 3486/ 2498/ 1909\n",
    "train = pd.read_csv('total_20210121.csv')\n",
    "train = train[['comments', 'hate']]\n",
    "train.columns = ['comments', 'label']\n",
    "train = kaggle_format(train)\n",
    "train = train.astype({'label': 'str'})\n",
    "\n",
    "# dev 데이터 / 471/ \n",
    "# 'none'/ 'offensive'/ 'hate' : 160/ 189/ 122\n",
    "dev = pd.read_csv('./korean-hate-speech-master/labeled/dev.tsv', sep='\\t')\n",
    "dev = dev[['comments', 'hate']]\n",
    "dev.columns = ['comments', 'label']\n",
    "dev = kaggle_format(dev)\n",
    "dev = dev.astype({'label': 'str'})\n",
    "\n",
    "test = pd.read_csv('./korean-hate-speech-master/test.no_label.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = train.comments, train.label\n",
    "X_test, y_test = dev.comments, dev.label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "vect_cv = CountVectorizer(min_df = 0.0, analyzer = 'char', \n",
    "    ngram_range = (1,3), max_features=5000) \n",
    "\n",
    "vect_tf = TfidfVectorizer(min_df = 0.0, analyzer = 'char', sublinear_tf=True, \n",
    "    ngram_range = (1,3), max_features=5000)\n",
    "\n",
    "X_cv = vect_cv.fit_transform(X)\n",
    "X_tf = vect_tf.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kf = KFold(n_splits=10)\n",
    "skfold = StratifiedKFold(n_splits=10)\n",
    "\n",
    "lgs = LogisticRegression(multi_class='multinomial', class_weight = 'balanced', random_state=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_f1_sk = cross_val_score(lgs, X_tf, y, scoring='f1_macro', cv=skfold)\n",
    "cv_f1_sk = cross_val_score(lgs, X_cv, y, scoring='f1_macro', cv=skfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_f1 = cross_val_score(lgs, X_tf, y, scoring='f1_macro', cv=kf)\n",
    "cv_f1 = cross_val_score(lgs, X_cv, y, scoring='f1_macro', cv=kf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### skfold kfold 둘다"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFold f1_macro :: TF :  [0.54409835 0.5751475  0.55495904 0.53784724 0.5603153  0.52843219\n",
      " 0.56598094 0.53796904 0.55379309 0.56143291] \n",
      "\n",
      "KFold f1_macro :: CV :  [0.54380155 0.54360996 0.53902544 0.53090814 0.54776016 0.52127145\n",
      " 0.55323764 0.54966987 0.53721223 0.52919632]\n",
      "\n",
      "KFold f1_macro :: TF :  0.5519975598739248 , CV :  0.5395692760168908\n"
     ]
    }
   ],
   "source": [
    "print('KFold f1_macro :: TF : ', tf_f1, '\\n\\nKFold f1_macro :: CV : ', cv_f1)\n",
    "print('\\nKFold f1_macro :: TF : ', np.mean(tf_f1), ', CV : ', np.mean(cv_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SKFold f1_macro :: TF :  [0.54163117 0.56082713 0.5771245  0.53130162 0.55013692 0.53510416\n",
      " 0.58301114 0.54874383 0.54916122 0.5544463 ] \n",
      "\n",
      "SKFold f1_macro :: CV :  [0.53854502 0.53605738 0.55034544 0.5346379  0.54060879 0.5322481\n",
      " 0.56368439 0.55686064 0.53945839 0.51794944]\n",
      "\n",
      "SKFold f1_macro :: TF :  0.5531487999460587 , CV :  0.5410395492065031\n"
     ]
    }
   ],
   "source": [
    "print('SKFold f1_macro :: TF : ', tf_f1_sk, '\\n\\nSKFold f1_macro :: CV : ', cv_f1_sk)\n",
    "print('\\nSKFold f1_macro :: TF : ', np.mean(tf_f1_sk), ', CV : ', np.mean(cv_f1_sk))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_f1_score(pre_data):\n",
    "    vec = TfidfVectorizer(min_df=0.0, analyzer='char', ngram_range=(1, 3), sublinear_tf=True,\n",
    "               max_features='5000')\n",
    "    X_tf = vec.fit_transform(pre_data)\n",
    "    print(X_tf.shape)\n",
    "    lgs = LogisticRegression(multi_class='multinomial', solver='lbfgs', C=1, \n",
    "                         class_weight='balanced', \n",
    "                         max_iter=6000, random_state=10)\n",
    "    lgs.fit(X_tf, y)\n",
    "    X_test_tf = vec.transform(X_test)\n",
    "    pred =  lgs.predict(X_test_tf)\n",
    "    score = f1_score(y_test, pred, average='macro')\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dev 파일 :: TF :  0.5773105429455988\n",
      "dev 파일 :: CV :  0.5594804815636172\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "lgs.fit(X_tf, y)\n",
    "X_test1 = vect_tf.transform(X_test)\n",
    "pred_tf = lgs.predict(X_test1)\n",
    "print('dev 파일 :: TF : ', f1_score(y_test, pred_tf, average='macro'))\n",
    "\n",
    "lgs.fit(X_cv, y)\n",
    "X_test2 = vect_cv.transform(X_test)\n",
    "pred_cv = lgs.predict(X_test2)\n",
    "print('dev 파일 :: CV : ', f1_score(y_test, pred_cv, average='macro'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}