sklearn - Feature Selection through Feature Scaling (Standardization) .ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection through Standardization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import the relevant libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For these lessons we will need NumPy, pandas, matplotlib and seaborn\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "\n",
    "# and of course the actual regression (machine learning) module\n",
    "from sklearn.linear_model import LinearRegression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAT</th>\n",
       "      <th>Rand 1,2,3</th>\n",
       "      <th>GPA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1714</td>\n",
       "      <td>1</td>\n",
       "      <td>2.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1664</td>\n",
       "      <td>3</td>\n",
       "      <td>2.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1760</td>\n",
       "      <td>3</td>\n",
       "      <td>2.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1685</td>\n",
       "      <td>3</td>\n",
       "      <td>2.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1693</td>\n",
       "      <td>2</td>\n",
       "      <td>2.83</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    SAT  Rand 1,2,3   GPA\n",
       "0  1714           1  2.40\n",
       "1  1664           3  2.52\n",
       "2  1760           3  2.54\n",
       "3  1685           3  2.74\n",
       "4  1693           2  2.83"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the data from a .csv in the same folder\n",
    "data = pd.read_csv(r'D:\\Stu\\Data Science\\Exercises Regression\\1.02. Multiple linear regression.csv')\n",
    "\n",
    "# Let's explore the top 5 rows of the df\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAT</th>\n",
       "      <th>Rand 1,2,3</th>\n",
       "      <th>GPA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>84.000000</td>\n",
       "      <td>84.000000</td>\n",
       "      <td>84.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1845.273810</td>\n",
       "      <td>2.059524</td>\n",
       "      <td>3.330238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>104.530661</td>\n",
       "      <td>0.855192</td>\n",
       "      <td>0.271617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1634.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.400000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1772.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.190000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1846.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.380000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1934.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.502500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2050.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.810000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               SAT  Rand 1,2,3        GPA\n",
       "count    84.000000   84.000000  84.000000\n",
       "mean   1845.273810    2.059524   3.330238\n",
       "std     104.530661    0.855192   0.271617\n",
       "min    1634.000000    1.000000   2.400000\n",
       "25%    1772.000000    1.000000   3.190000\n",
       "50%    1846.000000    2.000000   3.380000\n",
       "75%    1934.000000    3.000000   3.502500\n",
       "max    2050.000000    3.000000   3.810000"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the multiple linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Declare the dependent and independent variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are two independent variables: 'SAT' and 'Rand 1,2,3'\n",
    "x = data[['SAT','Rand 1,2,3']]\n",
    "\n",
    "# and a single dependent variable: 'GPA'\n",
    "y = data['GPA']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Standardization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import the preprocessing module\n",
    "# StandardScaler is one of the easiest and 'cleanest' ways to preprocess your data\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create an instance of the StandardScaler class\n",
    "scaler = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "StandardScaler(copy=True, with_mean=True, with_std=True)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the input data (x)\n",
    "# Essentially we are calculating the mean and standard deviation feature-wise \n",
    "# (the mean of 'SAT' and the standard deviation of 'SAT', \n",
    "# as well as the mean of 'Rand 1,2,3' and the standard deviation of 'Rand 1,2,3')\n",
    "scaler.fit(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The actual scaling of the data is done through the method 'transform()'\n",
    "# Let's store it in a new variable, named appropriately\n",
    "x_scaled = scaler.transform(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.26338288, -1.24637147],\n",
       "       [-1.74458431,  1.10632974],\n",
       "       [-0.82067757,  1.10632974],\n",
       "       [-1.54247971,  1.10632974],\n",
       "       [-1.46548748, -0.07002087],\n",
       "       [-1.68684014, -1.24637147],\n",
       "       [-0.78218146, -0.07002087],\n",
       "       [-0.78218146, -1.24637147],\n",
       "       [-0.51270866, -0.07002087],\n",
       "       [ 0.04548499,  1.10632974],\n",
       "       [-1.06127829,  1.10632974],\n",
       "       [-0.67631715, -0.07002087],\n",
       "       [-1.06127829, -1.24637147],\n",
       "       [-1.28263094,  1.10632974],\n",
       "       [-0.6955652 , -0.07002087],\n",
       "       [ 0.25721362, -0.07002087],\n",
       "       [-0.86879772,  1.10632974],\n",
       "       [-1.64834403, -0.07002087],\n",
       "       [-0.03150724,  1.10632974],\n",
       "       [-0.57045283,  1.10632974],\n",
       "       [-0.81105355,  1.10632974],\n",
       "       [-1.18639066,  1.10632974],\n",
       "       [-1.75420834,  1.10632974],\n",
       "       [-1.52323165, -1.24637147],\n",
       "       [ 1.23886453, -1.24637147],\n",
       "       [-0.18549169, -1.24637147],\n",
       "       [-0.5608288 , -1.24637147],\n",
       "       [-0.23361183,  1.10632974],\n",
       "       [ 1.68156984, -1.24637147],\n",
       "       [-0.4934606 , -0.07002087],\n",
       "       [-0.73406132, -1.24637147],\n",
       "       [ 0.85390339, -1.24637147],\n",
       "       [-0.67631715, -1.24637147],\n",
       "       [ 0.09360513,  1.10632974],\n",
       "       [ 0.33420585, -0.07002087],\n",
       "       [ 0.03586096, -0.07002087],\n",
       "       [-0.35872421,  1.10632974],\n",
       "       [ 1.04638396,  1.10632974],\n",
       "       [-0.65706909,  1.10632974],\n",
       "       [-0.13737155, -0.07002087],\n",
       "       [ 0.18984542,  1.10632974],\n",
       "       [ 0.04548499, -1.24637147],\n",
       "       [ 1.1618723 ,  1.10632974],\n",
       "       [-1.37887123, -1.24637147],\n",
       "       [ 1.39284898, -1.24637147],\n",
       "       [ 0.76728713, -0.07002087],\n",
       "       [-0.20473975, -0.07002087],\n",
       "       [ 1.06563201, -1.24637147],\n",
       "       [ 0.11285319, -1.24637147],\n",
       "       [ 1.28698467,  1.10632974],\n",
       "       [-0.41646838,  1.10632974],\n",
       "       [ 0.09360513, -1.24637147],\n",
       "       [ 0.59405462, -0.07002087],\n",
       "       [-2.03330517, -0.07002087],\n",
       "       [ 0.32458182, -1.24637147],\n",
       "       [ 0.40157405, -1.24637147],\n",
       "       [-1.10939843, -0.07002087],\n",
       "       [ 1.03675993, -1.24637147],\n",
       "       [-0.61857297, -0.07002087],\n",
       "       [ 0.44007016, -0.07002087],\n",
       "       [ 1.14262424, -1.24637147],\n",
       "       [-0.35872421,  1.10632974],\n",
       "       [ 0.45931822,  1.10632974],\n",
       "       [ 1.88367444,  1.10632974],\n",
       "       [ 0.45931822, -1.24637147],\n",
       "       [-0.12774752, -0.07002087],\n",
       "       [ 0.04548499,  1.10632974],\n",
       "       [ 0.85390339, -0.07002087],\n",
       "       [ 0.15134931, -0.07002087],\n",
       "       [ 0.8250313 ,  1.10632974],\n",
       "       [ 0.84427936,  1.10632974],\n",
       "       [-0.64744506, -1.24637147],\n",
       "       [ 1.24848856, -1.24637147],\n",
       "       [ 0.85390339,  1.10632974],\n",
       "       [ 1.69119387,  1.10632974],\n",
       "       [ 1.6334497 ,  1.10632974],\n",
       "       [ 1.46021718, -1.24637147],\n",
       "       [ 1.68156984, -0.07002087],\n",
       "       [-0.02188321,  1.10632974],\n",
       "       [ 0.87315144,  1.10632974],\n",
       "       [-0.33947615, -1.24637147],\n",
       "       [ 1.3639769 ,  1.10632974],\n",
       "       [ 1.12337618, -1.24637147],\n",
       "       [ 1.97029069, -0.07002087]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The result is an ndarray\n",
    "x_scaled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression with scaled features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a regression works in the exact same way\n",
    "reg = LinearRegression()\n",
    "\n",
    "# We just need to specify that our inputs are the 'scaled inputs'\n",
    "reg.fit(x_scaled,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.17181389, -0.00703007])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's see the coefficients\n",
    "reg.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.330238095238095"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# And the intercept\n",
    "reg.intercept_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a summary table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# As usual we can try to arrange the information in a summary table\n",
    "# Let's create a new data frame with the names of the features\n",
    "reg_summary = pd.DataFrame([['Bias'],['SAT'],['Rand 1,2,3']], columns=['Features'])\n",
    "\n",
    "# Then we create and fill a second column, called 'Weights' with the coefficients of the regression\n",
    "# Since the standardized coefficients are called 'weights' in ML, this is a much better word choice for our case\n",
    "# Note that even non-standardized coeff. are called 'weights' \n",
    "# but more often than not, when doing ML we perform some sort of scaling\n",
    "reg_summary['Weights'] = reg.intercept_, reg.coef_[0], reg.coef_[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>Weights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Bias</td>\n",
       "      <td>3.330238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SAT</td>\n",
       "      <td>0.171814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Rand 1,2,3</td>\n",
       "      <td>-0.007030</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Features   Weights\n",
       "0        Bias  3.330238\n",
       "1         SAT  0.171814\n",
       "2  Rand 1,2,3 -0.007030"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we have a pretty clean summary, which can help us make an informed decision about the importance of each feature\n",
    "reg_summary"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}