python-3.x - 内存错误无法分配 array.shape

Question

我正在使用python语言。我正在训练包含大约 9000 张图像的数据集。我使用预训练的神经网络 VGG16。现在的问题是内存错误：Unable to allocate array with shape (500,500,3) and data type unit 32

我试过 float 64, 32, 16,8 但是内存错误是一样的，我应该怎么做才能处理这个错误。以及使用 keras 的数据生成功能的数据并行性，但似乎我做错了什么

图书馆

from keras.preprocessing.image import ImageDataGenerator
from keras.models import model_from_json
from keras.applications import VGG16
import numpy as np
import glob
import os
import keras
from keras import backend as K
from PIL import Image
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from sklearn.model_selection import train_test_split
from keras import optimizers
import matplotlib.pyplot as plt

img_rows, img_cols = 500,500
channels=3
def load_labels(myDir):
    labels=[]
    fileList = glob.glob(myDir)
    for fname in fileList:
        fileName = os.path.basename(fname)
        curLabel = fileName.split("_")[0]
        labels.append(curLabel)
    return np.asarray(labels)    
def load_dataThreeChannel(myDir):
    images=[]
    fileList = glob.glob(myDir)    
  #  x = np.array([np.array(Image.open(fname)).flatten() for fname in fileList])
  #  x = np.array([np.array(Image.open(fname)) for fname in fileList])
    for fname in fileList:
        #print(fname)
        img = Image.open(fname)
        output = np.array(img.resize((img_rows,img_cols), Image.ANTIALIAS))
        #output = np.stack((output,)*3, -1)
        images.append(output)  
    x=np.asarray(images)
    print(x.shape)
    return x
myDir ="train_patches/*.png"
labels = load_labels(myDir)
data = load_dataThreeChannel(myDir)
#data = load_data(myDir)
# Data gen to avoid memory error
# create a data generator
datagen = ImageDataGenerator()
# load and iterate training dataset
train_it = datagen.flow_from_directory("train_patches/*.png", class_mode='binary', batch_size=64)
# confirm the iterator works
batchX, batchy = train_it.next()
print('Batch shape=%s, min=%.3f, max=%.3f' % (batchX.shape, batchX.min(), batchX.max()))

#Include_top=False, Does not load the last two fully connected layers which act as the classifier.
#We are just loading the convolutional layers. 
vgg_conv = VGG16(weights='imagenet',include_top=False,input_shape=(img_rows,img_cols,3))

# freeze the layer except the last 4 layers
for layer in vgg_conv.layers[:-4]:
    layer.trainable=False
num_classes=10
model = Sequential() 
# Add the vgg convolutional base model
model.add(vgg_conv) 
# Add new layers
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
 # Show a summary of the model. Check the number of trainable parameters
model.summary()    
epochs = 3
X_train = data
Y_train =labels
#X_train,X_test,Y_train,Y_test = train_test_split(data,labels, test_size=0.20, random_state=42)

X_train = X_train.astype('float32', copy= False)
#X_test = X_test.astype('float32', copy= False)
X_train /= 255
#X_test /= 255
print('x_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
#print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = keras.utils.to_categorical(Y_train, num_classes)
#Y_test = keras.utils.to_categorical(Y_test, num_classes)
# Compile the model
model.compile(loss='categorical_crossentropy',
              optimizer=optimizers.RMSprop(lr=1e-4),
              metrics=['acc'])
# Train the model
history = model.fit(
      X_train,Y_train,
      epochs=10,
      verbose=1)
acc = history.history['acc']
loss = history.history['loss']
epochs = range(len(acc))
plt.plot(epochs, acc, 'b', label='Training acc')
plt.title('Training accuracy')
plt.legend()
plt.show()
model_json = model.to_json()
open('imdata.json','w').write(model_json)
model.save_weights('imdata.h5',overwrite=True)

请帮我解决这个错误。还请解释一下 imdata.h5 文件以及 imdata_json 文件。

标签： python-3.xdeep-learningneural-network

Answer 1

# -*- coding: utf-8 -*-
"""
Please kindly check is this correct im still getting issues in 3 channels
"""

from keras.models import model_from_json
from keras.applications import VGG16
import numpy as np
import glob
import os
import keras
from keras import backend as K
from PIL import Image
from keras.models import Sequential
from keras.layers import Dense, Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D
from sklearn.model_selection import train_test_split
from keras import optimizers
import matplotlib.pyplot as plt
from keras.preprocessing.image import ImageDataGenerator



img_rows, img_cols = 500,500
channels=3

# def load_labels(myDir):
#     labels=[]
#     fileList = glob.glob(myDir)
#     for fname in fileList:
#         fileName = os.path.basename(fname)
#         curLabel = fileName.split("_")[0]
#         labels.append(curLabel)

#     return np.asarray(labels)   


def load_dataThreeChannel(myDir):
    images=[]
    fileList = glob.glob(myDir)    
  #  x = np.array([np.array(Image.open(fname)).flatten() for fname in fileList])
  #  x = np.array([np.array(Image.open(fname)) for fname in fileList])
    for fname in fileList:
        #print(fname)
        img = Image.open(fname)
        output = np.array(img.resize((img_rows,img_cols), Image.ANTIALIAS))
        #output = np.stack((output,)*3, -1)
        images.append(output)  
    x=np.asarray(images)
    print(x.shape)
    return x

myDir = ImageDataGenerator(rescale=1.0/255)
#included in our dependencies

myDir =myDir.flow_from_directory("C:/Users/iohan/.spyder-py3/IAM/train_patches/", target_size=(224,224), color_mode='rgb', batch_size=32, class_mode='categorical', shuffle=True)

Xbatch, Ybatch = myDir.next()

# myDir ="C:/Users/iohan/.spyder-py3/IAM/train_patches/*.png"
# labels = load_labels(myDir)
data = load_dataThreeChannel(myDir)
#data = load_data(myDir)



#Include_top=False, Does not load the last two fully connected layers which act as the classifier.
#We are just loading the convolutional layers. 
vgg_conv = VGG16(weights='imagenet',include_top=False,input_shape=(img_rows,img_cols,3))

# freeze the layer except the last 4 layers
for layer in vgg_conv.layers[:-4]:
    layer.trainable=False
num_classes=10
model = Sequential() 
# Add the vgg convolutional base model
model.add(vgg_conv) 
# Add new layers
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
 # Show a summary of the model. Check the number of trainable parameters
model.summary()    
epochs = 3
X_train = data
Y_train =labels
#X_train,X_test,Y_train,Y_test = train_test_split(data,labels, test_size=0.20, random_state=42)

X_train = X_train.astype('float32', copy= False)
#X_test = X_test.astype('float32', copy= False)
X_train /= 255
#X_test /= 255
print('x_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
#print(X_test.shape[0], 'test samples')


# convert class vectors to binary class matrices
Y_train = keras.utils.to_categorical(Y_train, num_classes)
#Y_test = keras.utils.to_categorical(Y_test, num_classes)


# Compile the model
model.compile(loss='categorical_crossentropy',
              optimizer=optimizers.RMSprop(lr=1e-4),
              metrics=['acc'])
# Train the model
# history = model.fit(
#       X_train,Y_train,
#       epochs=10,
#       verbose=1)

history= step_size_train=myDir.n//myDir.batch_size
model.fit_generator(generator=myDir,
                   steps_per_epoch=step_size_train,
                   epochs=10)


acc = history.history['acc']
loss = history.history['loss']


epochs = range(len(acc))

plt.plot(epochs, acc, 'b', label='Training acc')
plt.title('Training accuracy')
plt.legend()


plt.show()


model_json = model.to_json()
open('imdata.json','w').write(model_json)
model.save_weights("C:/Users/iohan/.spyder-py3/IAM/mdata.h5",overwrite=True)