python - 如何使用 Python 在 Keras 中为 ANN 绘制 10 倍交叉验证的 ROC 曲线?
问题描述
我只是想为 Keras 中 ANN 的 10 倍交叉验证的所有 10 个实验找到 ROC 图。我坚持了一周,找不到解决方案。有人可以帮忙吗?我已经尝试了来自 sklearn 的以下链接(https://scikit-learn.org/stable/auto_examples/model_selection/plot_roc_crossval.html)的代码,并想使用包装器在 sklearn 中使用 Keras 模型,但它显示错误。我在python中的代码:
## Creating NN in Keras
# Load libraries
import numpy as np
from keras import models
from keras import layers
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_classification
# Set random seed
np.random.seed(7)
#Create Function That Constructs Neural Network
# Create function returning a compiled network
def create_network():
# Start neural network
network = models.Sequential()
# Add fully connected layer with a ReLU activation function
network.add(layers.Dense(units=25, activation='relu', input_shape=(X.shape[1],)))
# Add fully connected layer with a ReLU activation function
network.add(layers.Dense(units=X.shape[1], activation='relu'))
# Add fully connected layer with a sigmoid activation function
network.add(layers.Dense(units=1, activation='sigmoid'))
# Compile neural network
network.compile(loss='binary_crossentropy', # Cross-entropy
optimizer='adam', # Root Mean Square Propagation
metrics=['accuracy']) # Accuracy performance metric
# Return compiled network
return network
###
#Wrap Function In KerasClassifier
# Wrap Keras model so it can be used by scikit-learn
neural_network = KerasClassifier(build_fn=create_network,
epochs=150,
batch_size=10,
verbose=0)
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm, datasets
from sklearn.metrics import auc
from sklearn.metrics import plot_roc_curve
from sklearn.model_selection import StratifiedKFold
n_samples, n_features = X.shape
# Add noisy features
random_state = np.random.RandomState(0)
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]
# #############################################################################
# Classification and ROC analysis
# Run classifier with cross-validation and plot ROC curves
cv = StratifiedKFold(n_splits=10)
classifier = neural_network
tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)
fig, ax = plt.subplots()
for i, (train, test) in enumerate(cv.split(X, y)):
classifier.fit(X[train], y[train])
viz = plot_roc_curve(classifier, X[test], y[test],
name='ROC fold {}'.format(i),
alpha=0.3, lw=1, ax=ax)
interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)
interp_tpr[0] = 0.0
tprs.append(interp_tpr)
aucs.append(viz.roc_auc)
ax.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
label='Chance', alpha=.8)
mean_tpr = np.mean(tprs, axis=0)
mean_tpr[-1] = 1.0
mean_auc = auc(mean_fpr, mean_tpr)
std_auc = np.std(aucs)
ax.plot(mean_fpr, mean_tpr, color='b',
label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
lw=2, alpha=.8)
std_tpr = np.std(tprs, axis=0)
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
ax.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
label=r'$\pm$ 1 std. dev.')
ax.set(xlim=[-0.05, 1.05], ylim=[-0.05, 1.05],
title="Receiver operating characteristic example")
ax.legend(loc="lower right")
plt.show()
**It shows the following error:**
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-29-f10078491154> in <module>()
40 viz = plot_roc_curve(classifier, X[test], y[test],
41 name='ROC fold {}'.format(i),
---> 42 alpha=0.3, lw=1, ax=ax)
43 interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr)
44 interp_tpr[0] = 0.0
/usr/local/lib/python3.6/dist-packages/sklearn/metrics/_plot/roc_curve.py in plot_roc_curve(estimator, X, y, sample_weight, drop_intermediate, response_method, name, ax, **kwargs)
170 )
171 if not is_classifier(estimator):
--> 172 raise ValueError(classification_error)
173
174 prediction_method = _check_classifer_response_method(estimator,
ValueError: KerasClassifier should be a binary classifier
解决方案
我有同样的问题。我发现这个链接非常有用。 https://www.kaggle.com/kanncaa1/roc-curve-with-k-fold-cv。我已针对我的情况对其进行了如下修改:
seed = 7
np.random.seed(seed)
tprs = []
aucs = []
mean_fpr = np.linspace(0, 1, 100)
i = 1
fig, ax = plt.subplots()
kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=seed)
# for i, (train, test) in enumerate(cv.split(X_13 , target)):
for train, test in kfold.split(X_train, y_train):
# create model
model= Sequential()
model.add(Dense(100, input_dim=X_train.shape[1], activation= 'relu',kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(80, activation = 'relu',kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(1, activation = 'sigmoid'))
##- compile model
sgd = SGD(lr=0.1, momentum=0.8)
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy'])
model.fit(X_train[train], y_train[train], epochs=100, batch_size=15,verbose=0)
# evaluate the model
y_pred_keras = model.predict_proba(X_train[test]).ravel()
fpr, tpr, thresholds = roc_curve(y_train[test], y_pred_keras)
tprs.append(interp(mean_fpr, fpr, tpr))
roc_auc = auc(fpr, tpr)
aucs.append(roc_auc)
plt.plot(fpr, tpr, lw=2, alpha=0.3, label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))
i= i+1
plt.plot([0,1],[0,1],linestyle = '--',lw = 2,color = 'black')
mean_tpr = np.mean(tprs, axis=0)
mean_auc = auc(mean_fpr, mean_tpr)
plt.plot(mean_fpr, mean_tpr, color='blue',
label=r'Mean ROC (AUC = %0.2f )' % (mean_auc),lw=2, alpha=1)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC')
plt.legend(loc="lower right")
plt.show()
希望它可以帮助!
推荐阅读
- asp.net-core - 作为 PWA 托管在 Ubuntu 上的 ASP.NET Core 出现错误“不安全清单”并阻止服务工作者
- android - 为什么这个方法在没有命中 return 语句的情况下返回?
- anaconda - “conda create -n something anaconda”是什么意思?
- react-native - 如何使用 react-navigation 重置到不同的堆栈
- python-3.x - 计算数据框列中用逗号分隔的数字的总和
- python - 使用 python 将嵌套 json 转换为平面 json 时遇到困难
- netlogo - 改变品种:如何判断品种是否真的在改变?
- azure - 在发布管道中找不到 Autofac
- c# - 一个类包含同一个类的对象的堆栈实现
- javascript - 图书馆项目 - 移除书籍时更新对象