Use of ROC_AUC in Classification Problem¶
In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
In [0]:
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.feature_selection import VarianceThreshold
In [4]:
data = pd.read_csv('/content/drive/My Drive/kaggle/Learning_mutual_information/data/santander-train.csv', nrows=20000)
data.head()
Out[4]:
In [5]:
x = data.drop('TARGET', axis=1)
y = data['TARGET']
x.shape, y.shape
Out[5]:
In [0]:
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0, stratify=y)
Remove Constant, Quasi Constant and Duplicate Features¶
In [0]:
# remove Constant and quasi constant features
constant_filter = VarianceThreshold(threshold=0.01)
constant_filter.fit(x_train)
x_train_filter = constant_filter.transform(x_train)
x_test_filter = constant_filter.transform(x_test)
In [8]:
x_train_filter.shape, x_test_filter.shape
Out[8]:
In [0]:
# remove duplicate features
x_train_T = x_train_filter.T
x_test_T = x_test_filter.T
In [0]:
x_train_T = pd.DataFrame(x_train_T)
x_test_T = pd.DataFrame(x_test_T)
In [11]:
x_train_T.duplicated().sum()
Out[11]:
In [0]:
duplicated_features = x_train_T.duplicated()
In [0]:
features_to_keep = [not index for index in duplicated_features]
x_train_unique = x_train_T[features_to_keep].T
x_test_unique = x_test_T[features_to_keep].T
In [14]:
x_train_unique.shape, x_train.shape
Out[14]:
Calculate ROC_AUC Score¶
In [0]:
# 独立変数一つづつ使ってRandomForestでモデリング後、roc_auc_scoreを使って変数毎のスコアを求める
roc_auc = []
for feature in x_train_unique.columns:
clf = RandomForestClassifier(n_estimators=100, random_state=0)
clf.fit(x_train_unique[feature].to_frame(), y_train)
y_pred = clf.predict(x_test_unique[feature].to_frame())
roc_auc.append(roc_auc_score(y_test, y_pred))
In [16]:
print(roc_auc)
In [0]:
roc_values = pd.Series(roc_auc)
roc_values.index = x_train_unique.columns
roc_values.sort_values(ascending=False, inplace=True)
In [18]:
roc_values
Out[18]:
In [19]:
roc_values.plot.bar()
Out[19]:
In [20]:
sel = roc_values[roc_values > 0.5]
sel
Out[20]:
In [0]:
x_train_roc = x_train_unique[sel.index]
x_test_roc = x_test_unique[sel.index]
Build Model and compare the performance¶
In [0]:
def run_randomForest(x_train, x_test, y_train, y_test):
clf = RandomForestClassifier(n_estimators=100,random_state=0, n_jobs=-1)
clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
print('Accuracy on test set: ', accuracy_score(y_test, y_pred))
In [23]:
%%time
run_randomForest(x_train_roc, x_test_roc, y_train, y_test)
In [24]:
x_train_roc.shape
Out[24]:
In [25]:
%%time
run_randomForest(x_train, x_test, y_train, y_test)
In [0]:
Feature Selection using RMSE in Regression¶
In [0]:
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression
In [0]:
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
In [28]:
boston = load_boston()
print(boston.DESCR)
In [29]:
x = pd.DataFrame(boston.data, columns=boston.feature_names)
x.head()
Out[29]:
In [0]:
y = boston.target
In [0]:
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
In [0]:
mse = []
for feature in x_train.columns:
clf = LinearRegression()
clf.fit(x_train[feature].to_frame(), y_train)
y_pred = clf.predict(x_test[feature].to_frame())
mse.append(mean_squared_error(y_test, y_pred))
In [33]:
mse
Out[33]:
In [35]:
mse = pd.Series(mse, index=x_train.columns)
mse.sort_values(ascending=False, inplace=True)
mse
# 結果：RM, LSTATの誤差が他の変数の半分なので、この２つの変数が重要
Out[35]:
In [36]:
mse.plot.bar()
Out[36]:
In [0]:
# 最も重要な２つの変数(RM, LSTAT)でモデリングし、結果を見る。
x_train_2 = x_train[['RM', 'LSTAT']]
x_test_2 = x_test[['RM', 'LSTAT']]
In [40]:
%%time
model = LinearRegression()
model.fit(x_train_2, y_train)
y_pred = model.predict(x_test_2)
print('r2_score: ', r2_score(y_test, y_pred))
print('rmse: ', np.sqrt(mean_squared_error(y_test, y_pred)))
print('sd of house price: ', np.std(y))
In [41]:
%%time
model = LinearRegression()
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
print('r2_score: ', r2_score(y_test, y_pred))
print('rmse: ', np.sqrt(mean_squared_error(y_test, y_pred)))
print('sd of house price: ', np.std(y))
In [0]: