python - 如何在 Keras.Model 中获得 l2 正则化损失值?
问题描述
我用keras.Model建模型,但是我用了自定义损失函数,自定义训练过程,我写了迭代过程和sess.run,然后我想在迭代过程中得到权重l2损失,怎么做它?
支持机型如下:
def model():
x = Input(shape=(None, None, 3))
y = Conv2D(10, 3, strides=1, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=regularizers.l2(0.0005))(x)
y = Conv2D(16, 3, strides=1, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=regularizers.l2(0.0005))(y)
y = Conv2D(32, 3, strides=1, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=regularizers.l2(0.0005))(y)
y = Conv2D(16, 3, strides=1, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=regularizers.l2(0.0005))(y)
y = Conv2D(1, 3, strides=1, kernel_initializer=tf.glorot_uniform_initializer(), kernel_regularizer=regularizers.l2(0.0005))(y)
return Model(inputs=[x], outputs=[y])
def loss(y_true, y_pred):
return tf.softmax_loss(.....)
火车代码:
def train():
dataset = tf.TFRecordDataset(tfrecords).make_one_shot_iterator().get_next()
input_image = tf.placeholder(...)
label = tf.placeholder(...)
net = model()
pred = model(input_image)
loss_op = loss(label, pred)
while True:
imgs, loss = sess.run([dataset, loss_op])
通过上面的代码,我不认为我已经减肥了。我怎么才能得到它?我尝试使用l2_loss_op = tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES))
,但值为 0。
解决方案
更新:更清洁和更完整的实现在这里。
我写了一个自定义函数来总结所有层的 return l1
、l2
和loss ,包括循环的;l1_l2
不包括损失,这不是activity_regularizer
重量损失:
def l1l2_weight_loss(model):
l1l2_loss = 0
for layer in model.layers:
if 'layer' in layer.__dict__ or 'cell' in layer.__dict__:
l1l2_loss += _l1l2_rnn_loss(layer)
continue
if 'kernel_regularizer' in layer.__dict__ or \
'bias_regularizer' in layer.__dict__:
l1l2_lambda_k, l1l2_lambda_b = [0,0], [0,0] # defaults
if layer.__dict__['kernel_regularizer'] is not None:
l1l2_lambda_k = list(layer.kernel_regularizer.__dict__.values())
if layer.__dict__['bias_regularizer'] is not None:
l1l2_lambda_b = list(layer.bias_regularizer.__dict__.values())
if any([(_lambda != 0) for _lambda in (l1l2_lambda_k + l1l2_lambda_b)]):
W = layer.get_weights()
for idx,_lambda in enumerate(l1l2_lambda_k + l1l2_lambda_b):
if _lambda != 0:
_pow = 2**(idx % 2) # 1 if idx is even (l1), 2 if odd (l2)
l1l2_loss += _lambda*np.sum(np.abs(W[idx//2])**_pow)
return l1l2_loss
def _l1l2_rnn_loss(layer):
l1l2_loss = 0
if 'backward_layer' in layer.__dict__:
bidirectional = True
_layer = layer.layer
else:
_layer = layer
bidirectional = False
ldict = _layer.cell.__dict__
if 'kernel_regularizer' in ldict or \
'recurrent_regularizer' in ldict or \
'bias_regularizer' in ldict:
l1l2_lambda_k, l1l2_lambda_r, l1l2_lambda_b = [0,0], [0,0], [0,0]
if ldict['kernel_regularizer'] is not None:
l1l2_lambda_k = list(_layer.kernel_regularizer.__dict__.values())
if ldict['recurrent_regularizer'] is not None:
l1l2_lambda_r = list(_layer.recurrent_regularizer.__dict__.values())
if ldict['bias_regularizer'] is not None:
l1l2_lambda_b = list(_layer.bias_regularizer.__dict__.values())
all_lambda = l1l2_lambda_k + l1l2_lambda_r + l1l2_lambda_b
if any([(_lambda != 0) for _lambda in all_lambda]):
W = layer.get_weights()
idx_incr = len(W)//2 # accounts for 'use_bias'
for idx,_lambda in enumerate(all_lambda):
if _lambda != 0:
_pow = 2**(idx % 2) # 1 if idx is even (l1), 2 if odd (l2)
l1l2_loss += _lambda*np.sum(np.abs(W[idx//2])**_pow)
if bidirectional:
l1l2_loss += _lambda*np.sum(
np.abs(W[idx//2 + idx_incr])**_pow)
return l1l2_loss
测试实施:
from keras.layers import Input, Dense, LSTM, GRU, Bidirectional
from keras.models import Model
from keras.regularizers import l1, l2, l1_l2
import numpy as np
ipt = Input(shape=(1200,16))
x = LSTM(60, activation='relu', return_sequences=True,
recurrent_regularizer=l2(1e-3),)(ipt)
x = Bidirectional(GRU(60, activation='relu', bias_regularizer =l1(1e-4)))(x)
out = Dense(1, activation='sigmoid', kernel_regularizer =l1_l2(2e-4))(x)
model = Model(ipt,out)
model.compile(loss='binary_crossentropy', optimizer='adam')
X = np.random.rand(10,1200,16) # (batch_size, timesteps, input_dim)
Y = np.random.randint(0,2,(10,1))
keras_loss = model.evaluate(X,Y)
custom_loss = binary_crossentropy(Y, model.predict(X))
custom_loss += l1l2_weight_loss(model)
print('%.6f'%keras_loss + ' -- keras_loss')
print('%.6f'%custom_loss + ' -- custom_loss')
0.763822 -- keras_loss 0.763822 -- custom_loss
(参见我对binary_crossentropy实现的回答)
推荐阅读
- android - 为什么在 Firemonkey 中启用 WordWrap 时 TText 修剪不起作用?
- php - 删除 jquery .click 上的特定 SQL 行。(我试图在单击时从数据库中删除图像)
- laravel - 发送变量以终止中间件
- php - 不同的反应
- rxjs - 我如何在一系列可观察数据中记住最后一个结果?
- here-api - 这里 api 没有在 /places/v1/diover/here 中显示特定纬度的任何数据
- selenium-webdriver - 如何使用“XPATH”处理内置关键字的 WebUI.click() 和 WebUI.setText()
- android - Firebase crashlytics 不显示崩溃
- go - 如何导入导入包的供应商包
- javascript - SVG:缩放 SVG 文档时如何缩放字体大小?