python - 卷积神经网络的反向传播
问题描述
我想只用 NumPy 库在 python 中编写我自己的卷积神经网络，所以我遵循了这两个教程：https : //victorzhou.com/blog/intro-to-cnns-part-1/，https://towardsdatascience .com/training-a-convolutional-neural-network-from-scratch-2235c2a25754关于 CNN 如何工作以及如何训练它，但由于某种原因，火车无法正常工作，没有人回复我对帖子的评论，我的数学老师也不明白。请在这里帮助我，我了解微积分，我所需要的只是清楚地解释为什么他和我的代码不起作用或反向传播如何工作。非常感谢任何和所有帮助。一个月后我有一个生成对抗性 CNN，我真的需要解决这个问题，谢谢你们所有人。
这是他的代码：
'''main.py'''
import numpy as np
from conv import Conv3x3
from maxpool import MaxPool2
from softmax import Softmax
# We only use the first 1k examples of each set in the interest of time.
# Feel free to change this if you want.
with np.load("mnist.npz") as mnist:
train_images = mnist["training_images"][:1000]
train_labels = mnist["training_labels"][:1000]
test_images = mnist["test_images"][:1000]
test_labels = mnist["test_labels"][:1000]
conv = Conv3x3(8) # 28x28x1 -> 26x26x8
pool = MaxPool2() # 26x26x8 -> 13x13x8
softmax = Softmax(13 * 13 * 8, 10) # 13x13x8 -> 10
def forward(image, label):
'''
Completes a forward pass of the CNN and calculates the accuracy and
cross-entropy loss.
- image is a 2d numpy array
- label is a digit
'''
# We transform the image from [0, 255] to [-0.5, 0.5] to make it easier
# to work with. This is standard practice.
out = conv.forward((image / 255) - 0.5)
out = pool.forward(out)
out = softmax.forward(out)
# Calculate cross-entropy loss and accuracy. np.log() is the natural log.
loss = -np.log(out[label])
acc = 1 if np.argmax(out) == label else 0
return out, loss, acc
def train(im, label, lr=.005):
'''
Completes a full training step on the given image and label.
Returns the cross-entropy loss and accuracy.
- image is a 2d numpy array
- label is a digit
- lr is the learning rate
'''
# Forward
out, loss, acc = forward(im, label)
# Calculate initial gradient
gradient = np.zeros(10)
gradient[label] = -1 / out[label]
# Backprop
gradient = softmax.backprop(gradient, lr)
gradient = pool.backprop(gradient)
gradient = conv.backprop(gradient, lr)
return loss, acc
print('MNIST CNN initialized!')
# Train the CNN for 3 epochs
for epoch in range(3):
print('--- Epoch %d ---' % (epoch + 1))
# Shuffle the training data
permutation = np.random.permutation(len(train_images))
train_images = train_images[permutation]
train_labels = train_labels[permutation]
# Train!
loss = 0
num_correct = 0
for i, (im, label) in enumerate(zip(train_images, train_labels)):
if i > 0 and i % 100 == 99:
print(
'[Step %d] Past 100 steps: Average Loss %.3f | Accuracy: %d%%' %
(i + 1, loss / 100, num_correct)
)
loss = 0
num_correct = 0
l, acc = train(im, label)
loss += l
num_correct += acc
# Test the CNN
print('\n--- Testing the CNN ---')
loss = 0
num_correct = 0
for im, label in zip(test_images, test_labels):
_, l, acc = forward(im, label)
loss += l
num_correct += acc
num_tests = len(test_images)
print('Test Loss:', loss / num_tests)
print('Test Accuracy:', num_correct / num_tests)
'''conv.py'''
import numpy as np
'''
Note: In this implementation, we assume the input is a 2d numpy array for simplicity, because that's
how our MNIST images are stored. This works for us because we use it as the first layer in our
network, but most CNNs have many more Conv layers. If we were building a bigger network that needed
to use Conv3x3 multiple times, we'd have to make the input be a 3d numpy array.
'''
class Conv3x3:
# A Convolution layer using 3x3 filters.
def __init__(self, num_filters):
self.num_filters = num_filters
# filters is a 3d array with dimensions (num_filters, 3, 3)
# We divide by 9 to reduce the variance of our initial values
self.filters = np.random.randn(num_filters, 3, 3) / 9
def iterate_regions(self, image):
'''
Generates all possible 3x3 image regions using valid padding.
- image is a 2d numpy array.
'''
h, w = image.shape
for i in range(h - 2):
for j in range(w - 2):
im_region = image[i:(i + 3), j:(j + 3)]
yield im_region, i, j
def forward(self, input):
'''
Performs a forward pass of the conv layer using the given input.
Returns a 3d numpy array with dimensions (h, w, num_filters).
- input is a 2d numpy array
'''
self.last_input = input
h, w = input.shape
output = np.zeros((h - 2, w - 2, self.num_filters))
for im_region, i, j in self.iterate_regions(input):
output[i, j] = np.sum(im_region * self.filters, axis=(1, 2))
return output
def backprop(self, d_L_d_out, learn_rate):
'''
Performs a backward pass of the conv layer.
- d_L_d_out is the loss gradient for this layer's outputs.
- learn_rate is a float.
'''
d_L_d_filters = np.zeros(self.filters.shape)
for im_region, i, j in self.iterate_regions(self.last_input):
for f in range(self.num_filters):
d_L_d_filters[f] += d_L_d_out[i, j, f] * im_region
# Update filters
self.filters -= learn_rate * d_L_d_filters
# We aren't returning anything here since we use Conv3x3 as the first layer in our CNN.
# Otherwise, we'd need to return the loss gradient for this layer's inputs, just like every
# other layer in our CNN.
return None
'''maxpool.py'''
import numpy as np
class MaxPool2:
# A Max Pooling layer using a pool size of 2.
def iterate_regions(self, image):
'''
Generates non-overlapping 2x2 image regions to pool over.
- image is a 2d numpy array
'''
h, w, _ = image.shape
new_h = h // 2
new_w = w // 2
for i in range(new_h):
for j in range(new_w):
im_region = image[(i * 2):(i * 2 + 2), (j * 2):(j * 2 + 2)]
yield im_region, i, j
def forward(self, input):
'''
Performs a forward pass of the maxpool layer using the given input.
Returns a 3d numpy array with dimensions (h / 2, w / 2, num_filters).
- input is a 3d numpy array with dimensions (h, w, num_filters)
'''
self.last_input = input
h, w, num_filters = input.shape
output = np.zeros((h // 2, w // 2, num_filters))
for im_region, i, j in self.iterate_regions(input):
output[i, j] = np.amax(im_region, axis=(0, 1))
return output
def backprop(self, d_L_d_out):
'''
Performs a backward pass of the maxpool layer.
Returns the loss gradient for this layer's inputs.
- d_L_d_out is the loss gradient for this layer's outputs.
'''
d_L_d_input = np.zeros(self.last_input.shape)
for im_region, i, j in self.iterate_regions(self.last_input):
h, w, f = im_region.shape
amax = np.amax(im_region, axis=(0, 1))
for i2 in range(h):
for j2 in range(w):
for f2 in range(f):
# If this pixel was the max value, copy the gradient to it.
if im_region[i2, j2, f2] == amax[f2]:
d_L_d_input[i * 2 + i2, j * 2 + j2, f2] = d_L_d_out[i, j, f2]
return d_L_d_input
'''softmax.py'''
import numpy as np
class Softmax:
# A standard fully-connected layer with softmax activation.
def __init__(self, input_len, nodes):
# We divide by input_len to reduce the variance of our initial values
self.weights = np.random.randn(input_len, nodes) / input_len
self.biases = np.zeros(nodes)
def forward(self, input):
'''
Performs a forward pass of the softmax layer using the given input.
Returns a 1d numpy array containing the respective probability values.
- input can be any array with any dimensions.
'''
self.last_input_shape = input.shape
input = input.flatten()
self.last_input = input
input_len, nodes = self.weights.shape
totals = np.dot(input, self.weights) + self.biases
self.last_totals = totals
exp = np.exp(totals)
return exp / np.sum(exp, axis=0)
def backprop(self, d_L_d_out, learn_rate):
'''
Performs a backward pass of the softmax layer.
Returns the loss gradient for this layer's inputs.
- d_L_d_out is the loss gradient for this layer's outputs.
- learn_rate is a float.
'''
# We know only 1 element of d_L_d_out will be nonzero
for i, gradient in enumerate(d_L_d_out):
if gradient == 0:
continue
# e^totals
t_exp = np.exp(self.last_totals)
# Sum of all e^totals
S = np.sum(t_exp)
# Gradients of out[i] against totals
d_out_d_t = -t_exp[i] * t_exp / (S ** 2)
d_out_d_t[i] = t_exp[i] * (S - t_exp[i]) / (S ** 2)
# Gradients of totals against weights/biases/input
d_t_d_w = self.last_input
d_t_d_b = 1
d_t_d_inputs = self.weights
# Gradients of loss against totals
d_L_d_t = gradient * d_out_d_t
# Gradients of loss against weights/biases/input
d_L_d_w = d_t_d_w[np.newaxis].T @ d_L_d_t[np.newaxis]
d_L_d_b = d_L_d_t * d_t_d_b
d_L_d_inputs = d_t_d_inputs @ d_L_d_t
# Update weights / biases
self.weights -= learn_rate * d_L_d_w
self.biases -= learn_rate * d_L_d_b
return d_L_d_inputs.reshape(self.last_input_shape)
这是我个人的改编：
当涉及到反向传播和梯度下降时，它们都不起作用。
'''main.py'''
import numpy as np
class conv_layer:
def __init__(self, num_filter, filter_dimention):
self.num_filter = num_filter
self.filter_dimention = filter_dimention
self.filters = np.random.randn(num_filter, filter_dimention,
filter_dimention) / filter_dimention**2
def iterate_regions(self, image):
h, w = image.shape
for i in range(h - (self.filter_dimention - 1)):
for j in range(w - (self.filter_dimention - 1)):
img_region = image[i:(i + self.filter_dimention),
j:(j + self.filter_dimention)]
yield img_region, i, j
def feedforward(self, input):
self.last_input = input
h, w = input.shape
output = np.zeros((h - (self.filter_dimention - 1),
w - (self.filter_dimention - 1), self.num_filter))
for img_region, i, j in self.iterate_regions(input):
output[i, j] = np.sum(img_region * self.filters, axis=(1, 2))
return output
def backprop(self, d_L_d_out, learn_rate):
d_L_d_filters = np.zeros(self.filters.shape)
for im_region, i, j in self.iterate_regions(self.last_input):
for f in range(self.num_filter):
d_L_d_filters[f] += d_L_d_out[i, j, f] * im_region
self.filters -= learn_rate * d_L_d_filters
return None
class max_pooling_layer:
def __init__(self, pool_size):
self.pool_size = pool_size
def iterate_regions(self, image):
h, w, _ = image.shape
new_h = h // self.pool_size
new_w = w // self.pool_size
for i in range(new_h):
for j in range(new_w):
img_region = image[(i * self.pool_size):(i * self.pool_size +
self.pool_size),
(j * self.pool_size):(j * self.pool_size +
self.pool_size)]
yield img_region, i, j
def feedforward(self, input):
self.last_input = input
h, w, num_filters = input.shape
output = np.zeros(
(h // self.pool_size, w // self.pool_size, num_filters))
for img_region, i, j in self.iterate_regions(input):
output[i, j] = np.amax(img_region, axis=(0, 1))
return output
def backprop(self, d_L_d_out):
d_L_d_input = np.zeros(self.last_input.shape)
for im_region, i, j in self.iterate_regions(self.last_input):
h, w, f = im_region.shape
amax = np.amax(im_region, axis=(0, 1))
for i2 in range(h):
for j2 in range(w):
for f2 in range(f):
if im_region[i2, j2, f2] == amax[f2]:
d_L_d_input[i * 2 + i2, j * 2 + j2, f2] = d_L_d_out[i, j, f2]
return d_L_d_input
class soft_max_layer:
def __init__(self, input_len, nodes):
self.weights = np.random.randn(input_len, nodes) / input_len
self.biases = np.zeros(nodes)
def feedforward(self, input):
self.last_input_shape = input.shape
input = input.flatten()
self.last_input = input
input_len, nodes = self.weights.shape
totals = np.dot(input, self.weights) + self.biases
self.last_totals = totals
exp = np.exp(totals)
return exp / np.sum(exp, axis=0)
def backprop(self, d_L_d_out, learn_rate):
for i, gradient in enumerate(d_L_d_out):
if gradient == 0:
pass
else:
S = np.sum(np.exp(d_L_d_out))
N = np.e**self.last_totals[i]
class Convolutional_Neural_Network:
def __init__(self, num_filter, filter_dimention, pool_size):
self.conv_layer = conv_layer(num_filter, filter_dimention)
self.max_pooling_layer = max_pooling_layer(pool_size)
self.soft_max_layer = soft_max_layer(13 * 13 * 8, 10)
def feedforward(self, input):
out = self.conv_layer.feedforward(input)
out = self.max_pooling_layer.feedforward(out)
out = self.soft_max_layer.feedforward(out)
return out
def calculate_accuracy(self, inputs, outputs):
accuracy = 0
for i, (input, output) in enumerate(zip(inputs, outputs)):
if np.argmax(self.feedforward(input)) == np.argmax(output):
accuracy += 1
accuracy /= len(inputs)
return accuracy
def train_network(self, inputs, outputs):
for image, output in zip(inputs, outputs):
output = output.astype(int)
out = self.feedforward((image / 255) - 0.5)
gradient = np.zeros(10)
gradient[output] = -1 / out[output]
gradient = self.soft_max_layer.backprop(gradient, 0.05)
gradient = self.max_pooling_layer.backprop(gradient)
gradient = self.conv_layer.backprop(gradient, 0.05)
with np.load('mnist.npz') as data:
train_size = 1
training_images = data['training_images'][:train_size]
training_images = np.reshape(training_images, (train_size, 28, 28))
training_labels = data['training_labels'][:train_size]
print("start....")
CNN = Convolutional_Neural_Network(8, 3, 2)
print(CNN.feedforward(training_images[0]).reshape((10,1)))
print(training_labels[0])
epochs = 5
for i in range(epochs):
CNN.train_network(training_images, training_labels)
accuracy = CNN.calculate_accuracy(training_images, training_labels)
print("\n" + "EPOCH " + str(i + 1) + " DONE | out of " + str(epochs) + " Accuracy: " + str(accuracy * 100) + "%")
print(CNN.feedforward(training_images[0]).reshape((10,1)))
print(training_labels[0])