In [1]:
import numpy as np
import pandas as pd
import matplotlib as plt
import seaborn as sns
In [0]:
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
In [0]:
from sklearn.feature_selection import VarianceThreshold, mutual_info_classif, mutual_info_regression
from sklearn.feature_selection import SelectKBest, SelectPercentile
In [4]:
data = pd.read_csv('/content/drive/My Drive/kaggle/Learning_mutual_information/data/santander-train.csv', nrows=20000)
data.head()
Out[4]:
In [5]:
x = data.drop('TARGET', axis=1)
y = data['TARGET']
x.shape, y.shape
Out[5]:
In [0]:
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0, stratify=y)
Remove constant, quasi constant, and duplicate features¶
In [0]:
# Remove constant, quasi constant features
constant_filter = VarianceThreshold(threshold=0.01)
constant_filter.fit(x_train)
x_train_filter = constant_filter.transform(x_train)
x_test_filter = constant_filter.transform(x_test)
In [0]:
# Remove duplicate features
x_train_T = x_train_filter.T
x_test_T = x_test_filter.T
In [0]:
x_train_T = pd.DataFrame(x_train_T)
x_test_T = pd.DataFrame(x_test_T)
In [11]:
# check duplicated numbers
x_train_T.shape
Out[11]:
In [12]:
x_train_T.duplicated().sum()
Out[12]:
In [15]:
duplicated_features = x_train_T.duplicated()
duplicated_features
Out[15]:
In [0]:
features_to_keep = [not index for index in duplicate_features]
In [0]:
x_train_unique = x_train_T[features_to_keep].T
x_test_unique = x_test_T[features_to_keep].T
In [18]:
x_train_unique.shape, x_test_unique.shape
Out[18]:
Calculate the Mutual Information¶
In [0]:
mi = mutual_info_classif(x_train_unique, y_train)
In [20]:
len(mei)
In [21]:
mi
Out[21]:
In [0]:
mi = pd.Series(mi)
mi.index = x_train_unique.columns
In [0]:
mi.sort_values(ascending=False, inplace=True)
In [26]:
mi.plot.bar(figsize=(16,5))
Out[26]:
In [27]:
# select features by percentile
# percentile=10 で 上位10パーセントの特徴量を選択
sel = SelectPercentile(mutual_info_classif, percentile=10).fit(x_train_unique, y_train)
x_train_unique.columns[sel.get_support()]
Out[27]:
In [28]:
len(x_train_unique.columns[sel.get_support()])
Out[28]:
In [29]:
# help document check
help(sel)
In [0]:
x_train_mi = sel.transform(x_train_unique)
x_test_mi = sel.transform(x_test_unique)
In [31]:
x_train_mi.shape
Out[31]:
Build the model and compare the performance¶
In [0]:
def run_randomForest(x_train, x_test, y_train, y_test):
clf = RandomForestClassifier(n_estimators=100, random_state=0, n_jobs=-1)
clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
print('Accuracy on test set: ')
print(accuracy_score(y_test, y_pred))
In [33]:
%%time
run_randomForest(x_train_mi, x_test_mi, y_train, y_test)
In [34]:
%%time
run_randomForest(x_train, x_test, y_train, y_test)
In [0]:
Mutual Information Gain in Regression¶
In [0]:
from sklearn.datasets import load_boston
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
In [0]:
boston = load_boston()
In [37]:
print(boston.DESCR)
In [38]:
x = pd.DataFrame(data=boston.data, columns=boston.feature_names)
x.head()
Out[38]:
In [0]:
y = boston.target
In [0]:
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
In [0]:
mi = mutual_info_regression(x_train, y_train)
mi = pd.Series(mi)
mi.index = x_train.columns
mi.sort_values(ascending=False, inplace=True)
In [42]:
mi
Out[42]:
In [43]:
mi.plot.bar()
Out[43]:
In [44]:
# get top 9 features
# k=9 <- get 9 features
sel = SelectKBest(mutual_info_regression, k=9).fit(x_train, y_train)
x_train.columns[sel.get_support()]
Out[44]:
In [0]:
model = LinearRegression()
model.fit(x_train, y_train)
y_predict = model.predict(x_test)
In [46]:
r2_score(y_test, y_predict)
Out[46]:
In [47]:
np.sqrt(mean_squared_error(y_test, y_predict))
Out[47]:
In [48]:
np.std(y)
Out[48]:
In [0]:
In [49]:
x_train_9 = sel.transform(x_train)
x_train_9.shape
Out[49]:
In [0]:
x_test_9 = sel.transform(x_test)
In [51]:
model = LinearRegression()
model.fit(x_train_9, y_train)
y_predict = model.predict(x_test_9)
print('r2_score')
r2_score(y_test, y_predict)
Out[51]:
In [52]:
print('rmse')
np.sqrt(mean_squared_error(y_test, y_predict))
Out[52]:
In [0]: