python - 训练集和验证集混合在图像分类任务上

Question

我是在做图像分类的，但是在训练的时候，输入的训练集和验证集出现了混合，我通过tensorboard看到的，不知道问题出在哪里。

https://github.com/a87871660/Picture_classification 这是我的代码，不知道是不是线程有问题。

import os
import numpy as np
from PIL import Image
import tensorflow as tf
import matplotlib.pyplot as plt
from numpy import *

C = []
label_C = []
R = []
label_R = []
H = []
label_H = []
L = []
label_L = []
Y = []
label_Y = []

def get_file (file_dir):
    for file_1 in os.listdir(file_dir + '\\C'):
        C.append(file_dir + '\\C' + '\\' + file_1)
        label_C.append(0)
    for file_2 in os.listdir(file_dir + '\\R'):
        R.append(file_dir + '\\R' + '\\' +  file_2)
        label_R.append(1)
    for file_3 in os.listdir(file_dir + '\\H'):
        H.append(file_dir + '\\H' + '\\' + file_3)
        label_H.append(2)
    for file_4 in os.listdir(file_dir + '\\L'):
        L.append(file_dir + '\\L' + '\\' +  file_4)
        label_L.append(3)
    for file_5 in os.listdir(file_dir + '\\Y'):
        Y.append(file_dir + '\\Y' + '\\' +  file_5)
        label_Y.append(4)
    print('There are %d C\n There are %d R\n There are %d H\n There are %d L\n There are %d Y' %(len(C), len(R), len(H), len(L), len(Y)))

    image_list = np.hstack((C, R, H, L, Y))
    label_list = np.hstack((label_C, label_R, label_H, label_L, label_Y))

    temp = np.array([image_list, label_list])

    # translate [file_dir, label], [file_dir, label]...
    temp = temp.transpose()

    np.random.shuffle(temp)

    image_list = list(temp[:, 0])
    label_list = list(temp[:, 1])
    label_list = [int(float(i)) for i in label_list]

    """
    n_sample = len(label_list)
    n_val = int(math.ceil(n_sample * ratio))
    n_train = n_sample - n_val
    tra_iamges = image_list[0:n_train]
    tra_labels = label_list[0:n_train]
    tra_labels = [int(float(i)) for i in tra_labels]
    val_images = image_list[n_train:-1]
    val_labels = label_list[n_train:-1]
    val_labels = [int(float(i)) for i in val_labels]
    """

    return image_list, label_list

def get_batch(image, label, image_h, image_w, target_height, target_width, batch_size, capacity):
    image = tf.cast(image, tf.string)
    label = tf.cast(label, tf.int32)
    input_queue = tf.train.slice_input_producer([image, label], num_epochs=None, shuffle=False)
    label = input_queue[1]
    # Tensor("input_producer/GatherV2_1:0", shape=(), dtype=int32)
    image_contants = tf.read_file(input_queue[0])
    # Tensor("ReadFile:0", shape=(), dtype=string)
    image = tf.image.decode_jpeg(image_contants, channels=3)
    image = tf.image.convert_image_dtype(image, dtype=tf.float32)#!!
    image = tf.image.resize_images(image, [image_h, image_w])
    image = tf.image.resize_image_with_crop_or_pad(image, target_height, target_width)
    #image = tf.image.per_image_standardization(image)
    image_batch, label_batch = tf.train.batch([image, label], batch_size=batch_size, num_threads=16, capacity=capacity)
    #image_batch, label_batch = tf.train.shuffle_batch([image, label], batch_size=batch_size, capacity=capacity, min_after_dequeue=capacity - 1)
    label_batch = tf.reshape(label_batch, [batch_size])
    image_batch = tf.cast(image_batch, tf.float32)#??
    return image_batch, label_batch

def valuation_batch(val_iamge, val_label, image_h, image_w, target_height, target_width, batch_size, capacity):
    val_iamge = tf.cast(val_iamge, tf.string)
    val_label = tf.cast(val_label, tf.int32)
    input_queue = tf.train.slice_input_producer([val_iamge, val_label], num_epochs=None, shuffle=False)
    val_label = input_queue[1]
    image_contents = tf.read_file(input_queue[0])
    val_iamge = tf.image.decode_jpeg(image_contents, channels=3)
    val_iamge = tf.image.resize_images(val_iamge, [image_h, image_w])
    val_iamge = tf.image.resize_image_with_crop_or_pad(val_iamge, target_height, target_width)#!!
    image_batch, label_batch = tf.train.batch([val_iamge, val_label], batch_size=batch_size, num_threads=16, capacity=capacity)
    val_label_batch = tf.reshape(label_batch, [batch_size])
    val_image_batch = tf.cast(image_batch, tf.float32)
    return val_image_batch, val_label_batch

上面是预加载数据的代码，下面是训练代码。

import os
import numpy as np
import tensorflow as tf
from pre import get_file, get_batch, valuation_batch
from nw import deep_CNN, losses, trainning, evaluation

n_class = 5
image_h = 260
image_w = 220
target_height = 300
target_width = 260
batch_size = 16
capacity = 256
capacity1 = 256
max_step = 200000
learning_rate = 0.001
train_dir = 'E:\\DeepLearn\\picture_train'
valuation_dir = 'E:\\DeepLearn\\picture_valuation'
logs_train_dir = 'E:\\DeepLearn\\Classify_picture\\self\\model\\train\\'
logs_valuation_dir = 'E:\\DeepLearn\\Classify_picture\\self\\model\\valuation\\'

train, train_label = get_file(train_dir)
valuation, valuation_label = get_file(valuation_dir)
train_batch, train_label_batch = get_batch(train, train_label, image_h, image_w, target_height, target_width, batch_size, capacity)
val_image_batch, val_label_batch = valuation_batch(valuation, valuation_label, target_height, target_width, batch_size, capacity1)

is_training = tf.placeholder(tf.bool, shape=(), name='is_training')
image_batch = tf.cond(is_training, lambda: train_batch, lambda: val_image_batch)
label_batch = tf.cond(is_training, lambda: train_label_batch, lambda: val_label_batch)

train_logits = deep_CNN(image_batch, batch_size, n_class)
train_loss = losses(train_logits, label_batch)
train_op = trainning(train_loss, learning_rate)
train_acc = evaluation(train_logits, label_batch)

summary_op = tf.summary.merge_all()
sess = tf.Session()
train_writer = tf.summary.FileWriter(logs_train_dir, sess.graph)
valuation_writer = tf.summary.FileWriter(logs_valuation_dir)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())

coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)

try:
    for step in np.arange(max_step):
        if coord.should_stop():
            break

        _, summary_training, tra_loss, tra_acc= sess.run([train_op, summary_op, train_loss, train_acc], feed_dict={is_training: True})
        summary_valuation, val_loss, val_acc = sess.run([summary_op, train_loss, train_acc], feed_dict={is_training: False})

        if step % 100 == 0:
            print('Step %d, train loss = %.3f, valuation loss = %.3f, train accuracy = %.2f%%, valuation accuracy = %.2f%%, learning rate is %f' % (step, tra_loss, val_loss, tra_acc * 100.0, val_acc * 100.0, _[1]))
            checkpoint_path = os.path.join(logs_train_dir, 'model.ckpt')
            saver.save(sess, checkpoint_path, global_step=step)
            train_writer.add_summary(summary_training, step)
            valuation_writer.add_summary(summary_valuation, step)

except tf.errors.OutOfRangeError:
    print('Done training -- epoch limit reached')
finally:
    coord.request_stop()
coord.join(threads)
sess.close()

标签： pythontensorflow