python - 重塑的输入与理想不同[重塑的输入是具有 x 值的张量，但需要 y 值]

Question

我正在尝试使用一堆 720*480 的图像来训练我的模型。这个错误是在图像的预处理中，就像在 TrainModelscript 的上半部分一样。例如图像

所以出错的部分应该是这部分：如果我没有错。哈哈哈。batch_features, batch_labels = iterator.get_next()

但进一步调查表明，错误有点不在那条线上？作为 print()，没有输出。我不知道如何调试这个。因为我是 ML 新手，并且处理 cv 到数组或元组。

因此，回到我的 tfrecord。

createTFrecordscript:

import tensorflow as tf
import os ,glob
import argparse
import random


os.environ["CUDA_VISIBLE_DEVICES"]="0"
#func for 2feature
def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))

def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
#func for create a tf example with 2 feature
def _create_tfexample(image_data, label):
    example = tf.train.Example(features=tf.train.Features(feature={
            'image': _bytes_feature(image_data),
            'label': _int64_feature(label)
            }))
    return example

#not sure,create a id for each file?
def enumerate_filestoId(file_list, sort=True):
    file_ids = {}
    file_id_counter = 0

    if sort:
        file_list.sort()

    for file_name in file_list:
        if file_name not in file_ids:
            file_ids[class_name] = file_id_counter
            file_id_counter += 1

    return file_ids

#main func
def create_tfrecords(save_dir, dataset_name, filenames, class_ids, num_shards):

    im_per_shard = int( len(filenames) / num_shards ) + 1

    for shard in range(num_shards):
        output_filename = os.path.join(save_dir, '{}_{:03d}-of-{:03d}.tfrecord'
                                       .format(dataset_name, shard, num_shards))
        print('Writing into {}'.format(output_filename))
        filenames_shard = filenames[shard*im_per_shard:(shard+1)*im_per_shard]

        with tf.io.TFRecordWriter(output_filename) as tfrecord_writer:

            for filename in filenames_shard:
                image =  tf.gfile.GFile(filename, 'rb').read()
                class_name = os.path.basename(os.path.dirname(filename))
                label = class_ids[class_name]

                example = _create_tfexample(image, label)
                tfrecord_writer.write(example.SerializeToString())

    print('Finished writing {} images into TFRecords'.format(len(filenames)))

"""
def _is_png(filenames):
  Determine if a file is or is not PNG format image.
  Args:
    filename: string, path of the image file.
  Returns:
    boolean indicating if the image is a PNG.

  return '.png' in filenames
"""

def main(args):
    #find filenameadd it to a list
    supported_formats = ['*.jpg', '*.JPG', '*.jpeg', '*.JPEG']
    filenames = []
    for extension in supported_formats:
        pattern = os.path.join(args.input_dir, '**', extension)
        filenames.extend(glob.glob(pattern, recursive=False))
    #for random them?
    random.seed(args.seed)
    random.shuffle(filenames)

    num_test = int(args.split_ratio * len(filenames))
    num_shards_test = int(args.split_ratio * args.num_shards)
    num_shards_train = args.num_shards - num_shards_test

    # write the list of classes and their corresponding ids to a file
    file_list = [name for name in os.listdir(args.input_dir)
                    if os.path.isdir(os.path.join(args.input_dir, name))]
    file_ids = enumerate_filestoId(file_list)
    with open(os.path.join(args.output_dir, 'classes.txt'), 'w') as f:
        for cid in file_ids:
            print('{}:{}'.format(file_ids[cid], cid), file=f)

    # create TFRecords for the training and test sets
    create_tfrecords(save_dir=args.output_dir,
                     dataset_name='train',
                     filenames=filenames[num_test:],
                     file_ids=file_ids,
                     num_shards=num_shards_train)
    create_tfrecords(save_dir=args.output_dir,
                     dataset_name='test',
                     filenames=filenames[:num_test],
                     file_ids=file_ids,
                     num_shards=num_shards_test)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_dir', type=str,
                        help='path to the directory where the images will be read from')
    parser.add_argument('--output_dir', type=str,
                        help='path to the directory where the TFRecords will be saved to')
    parser.add_argument('--num_shards', type=int,
                        help='total number of shards')
    parser.add_argument('--split_ratio', type=float, default=0.2,
                        help='ratio of number of images in the test set to the total number of images')
    parser.add_argument('--seed', type=int, default=42,
                        help='random seed for repeatable train/test splits')
    args, unknown = parser.parse_known_args()
    main(args)

错误：

2019-11-30 11:41:22.526045: W tensorflow/core/framework/op_kernel.cc:1651] OP_REQUIRES failed at iterator_ops.cc:929 : Invalid argument: {{function_node __inference_Dataset_map_parse_function_29}} Input to reshape is a tensor with 97934 values, but the requested shape has 1036800
         [[{{node Reshape}}]]
Traceback (most recent call last):
  File "TrainEvalSaveModel2.py", line 134, in <module>
    batch_features, batch_labels,Runcounter=imgs_input_fn(tfRecordspath, perform_shuffle=False, repeat_count=1,batch_size=4,Runcounter=0)
  File "TrainEvalSaveModel2.py", line 51, in imgs_input_fn
    batch_features, batch_labels = iterator.get_next()#disabled for image direct to tensor
  File "E:\Coding\PYFastCache\PYVenv\Venv37_gpuacc\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 737, in get_next
    return self._next_internal()
  File "E:\Coding\PYFastCache\PYVenv\Venv37_gpuacc\lib\site-packages\tensorflow_core\python\data\ops\iterator_ops.py", line 651, in _next_internal
    output_shapes=self._flat_output_shapes)
  File "E:\Coding\PYFastCache\PYVenv\Venv37_gpuacc\lib\site-packages\tensorflow_core\python\ops\gen_dataset_ops.py", line 2691, in iterator_get_next_sync
    _six.raise_from(_core._status_to_exception(e.code, message), None)
  File "<string>", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: {{function_node __inference_Dataset_map_parse_function_29}} Input to reshape is a tensor with 97934 values, but the requested shape has 1036800
         [[{{node Reshape}}]] [Op:IteratorGetNextSync]

训练模型脚本：

from __future__ import absolute_import, division, print_function, unicode_literals
import tensorflow as tf
import sys
import cProfile
import os

Runcounter=0
tf.compat.v1.enable_eager_execution()


def imgs_input_fn(tfRecordspath, perform_shuffle=False, repeat_count=1, batch_size=4,Runcounter=0):
    def parse_function(serialized):
        features = {
            'image': tf.io.FixedLenFeature([], tf.string),
            'label': tf.io.FixedLenFeature([], tf.int64)
        }
        # Parse the serialized data so we get a dict with our data.
        parsed_example = tf.io.parse_single_example(serialized,features)
        Image_ypix=720
        Image_xpix=480
        input_name="Kakoii"
        # Get the image as raw bytes.
        image_shape = tf.stack([Image_ypix,Image_xpix, 3]) #,if not flatten,on this
        image_raw = parsed_example['image']
        label = tf.cast(parsed_example['label'], tf.float32)
        # Decode the raw bytes so it becomes a tensor with type.
        image = tf.decode_raw(image_raw, tf.uint8)
        image = tf.cast(image, tf.float32)
        image = tf.reshape(image,image_shape)
        #image = tf.reshape(image,-1),flatten
        #image = tf.subtract(image, pixel_val?) # Zero-center by mean pixel,too complex
        image = tf.reverse(image, axis=[2]) # 'RGB'->'BGR'
        #Yourimage = tf.reverse(image, axis=[2]) # 'RGB'->'BGR'
        tensor = dict(zip([input_name], [image])), [label] #onlycomplicate it
        #print(image)
        #print(tensor),debug use
        #tensor=Yourimage
        return tensor
    mydataset = tf.data.TFRecordDataset(tfRecordspath)
    # Parse the serialized data in the TFRecords files.
    # This returns TensorFlow tensors for the image and labels.
    tensor = mydataset.map(parse_function)
    if perform_shuffle:
        # Randomizes input using a window of 256 elements (read into memory)
        tensor = tensor.shuffle(buffer_size=480)
    tensor = tensor.repeat(repeat_count)  # Repeats dataset this # times
    tensor = tensor.batch(batch_size)  # Batch size to use
    iterator = tf.compat.v1.data.make_one_shot_iterator(tensor)
    #batch_features=iterator # directly image to tensor only
    #batch_labels=iterator #for image to tensor only
    batch_features, batch_labels = iterator.get_next()#disabled for image direct to tensor
    #Runcounter=1,removed sys
    #print(batch_features)
    #print(batch_labels)
    return batch_features, batch_labels,Runcounter

def Proceedprocess(tfRecordspath,batch_features, batch_labels):
#usesruncounter to check,removed
    exein=input("continue?(exit,savemodel,yes):")
    if exein=="yes":
        NNTrainEvalBuild(tfRecordspath,batch_features, batch_labels)
    elif exein=="exit":
        sys.exit()
    elif exein=="savemodel":
        savemodel()
    else:
        print("assume can proceed trainmodel")
        NNTrainEvalBuild(tfRecordspath,batch_features, batch_labels)


#a old low level api func in broken save file,removed 


def convertTuple(yourtup): 
    finalstr =  ''.join(yourtup) 
    return finalstr



def NNTrainEvalBuild(tfRecordspath,batch_features, batch_labels):
    print("start model&preprocess")
    batch_size=8
    N_EPOCHS=8
    N_TRAIN=8
    N_VALIDATION=16

    convertTuple(batch_features)# direct image to tensoroff
    convertTuple(batch_labels)

    """somee old code only in broken code file"""

    inputs1 = tf.keras.Input(shape=(3,),tensor=(batch_features,batch_labels))
    out1 = tf.keras.layers.Dense(1024, activation=tf.nn.relu)(inputs1)
    in2= tf.keras.layers.LeakyReLU()(out1)
    out2 = tf.keras.layers.Dense(5, activation=tf.nn.softmax)(in2)
    model = tf.keras.Model(inputs=inputs1, outputs=out2)

    #with planned,create it
    model.compile(optimizer=tf.compat.v1.train.AdamOptimizer(),
    loss='binary_crossentropy',
    metrics=['accuracy'])

    #statrt train
    tf.keras.Model.fit(train_iterator,
    epochs=N_EPOCHS,
    steps_per_epoch=N_TRAIN // batch_size,
    validation_data=val_iterator,
    validation_steps=N_VALIDATION // batch_size)
    print("finish training")

def savemodel():
    #createmodel
    savemodelpath=input("Save Model's path:")
    tf.keras.models.save_model(savemodelpath,save_format="tf")


tfRecordspath=input("TFRecords's path:")
batch_features, batch_labels,Runcounter=imgs_input_fn(tfRecordspath, perform_shuffle=False, repeat_count=1,batch_size=4,Runcounter=0)

Proceedprocess(tfRecordspath,batch_features, batch_labels)

标签： pythontensorflowtensortfrecord