tensorflow - FailedPreconditionError: FailedPr...onError()

Question

我FailedPreconditionError在运行 sess 时有。我的网络有两个不同的部分，pretrained-network并且新添加了Recognition network.

Pretrained model用于提取特征，该特征用于再次训练recognition。

在我的代码中，首先加载预训练模型。

    graph = tf.Graph()
    with graph.as_default():       
       input_data, input_labels, input_boxes = input_train_data.input_fn()


       input_boxes = tf.reshape(input_boxes,[input_boxes.shape[0]*2,-1])#convert from Nx8 to 2Nx4
       # build model and loss
       net = Net(input_data, is_training = False)
       f_saver = tf.train.Saver(max_to_keep=1000, write_version=tf.train.SaverDef.V2, save_relative_paths=True)
       sess_config = tf.ConfigProto(log_device_placement = False, allow_soft_placement = True)
       if FLAGS.gpu_memory_fraction < 0:
          sess_config.gpu_options.allow_growth = True
       elif FLAGS.gpu_memory_fraction > 0:
          sess_config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction;
       session = tf.Session(graph=graph, config=sess_config)

       tf.logging.info('Initialize from: ' + config.train.init_checkpoint)
       f_saver.restore(session, config.train.init_checkpoint)

f_saver 恢复预训练的模型。

然后conv5_3提取特征并将其输入Recognition网络。

       conv5_3 = net.end_points['conv5_3']
       with tf.variable_scope("Recognition"):
          global_step_rec = tf.Variable(0, name='global_step_rec', trainable=False)
          #Pass through recognition net
          r_net = regnet.ConstructRecNet(conv5_3)
          conv7_7 = r_net.end_points['pool7']
          #implement ROI Pooling
          #input boxes be in x1, y1, x2, y2
          h_fmap = tf.dtypes.cast(tf.shape(conv7_7)[1],tf.float32)
          w_fmap = tf.dtypes.cast(tf.shape(conv7_7)[2],tf.float32)
          #remap boxes at input images to feature mats
          #input_boxes = input_boxes / tf.constant([config.train.input_shape[0], config.train.input_shape[0],\
          #                                        config.train.input_shape[0], config.train.input_shape[0]], dtype=tf.float32)#Normalize with image size first
          remap_boxes=tf.matmul(input_boxes,tf.diag([w_fmap,h_fmap,w_fmap,h_fmap]))
          #put first column with image indexes
          rows = tf.expand_dims(tf.range(remap_boxes.shape[0]), 1)/2
          add_index = tf.concat([tf.cast(rows,tf.float32),remap_boxes],-1)
          index = tf.not_equal(tf.reduce_sum(add_index[:,4:],axis=1),0)
          remap_boxes = tf.gather_nd(add_index,tf.where(index))

          remap_boxes=tf.dtypes.cast(remap_boxes,tf.int32)
          prob = roi_pooling(conv7_7, remap_boxes, pool_height=1, pool_width=28)
          #Get features for CTC training
          prob = tf.transpose(prob, (1, 0, 2))  # prepare for CTC
          data_length = tf.fill([tf.shape(prob)[1]], tf.shape(prob)[0])  # input seq length, batch size
          ctc = tf.py_func(CTCUtils.compute_ctc_from_labels, [input_labels], [tf.int64, tf.int64, tf.int64])
          ctc_labels = tf.to_int32(tf.SparseTensor(ctc[0], ctc[1], ctc[2]))
          predictions = tf.to_int32(tf.nn.ctc_beam_search_decoder(prob, data_length, merge_repeated=False, beam_width=10)[0][0])
          tf.sparse_tensor_to_dense(predictions, default_value=-1, name='d_predictions')
          tf.reduce_mean(tf.edit_distance(predictions, ctc_labels, normalize=False), name='error_rate')
          loss = tf.reduce_mean(tf.compat.v1.nn.ctc_loss(inputs=prob, labels=ctc_labels, sequence_length=data_length, ctc_merge_repeated=True), name='loss')
          learning_rate = tf.train.piecewise_constant(global_step_rec, [150000, 200000],[config.train.learning_rate, 0.1 * config.train.learning_rate,0.01 * config.train.learning_rate])
          opt_loss = tf.contrib.layers.optimize_loss(loss, global_step_rec, learning_rate, config.train.opt_type,config.train.grad_noise_scale, name='train_step')
          tf.global_variables_initializer() 

我可以运行 sess 直到特征提取 conv5_3。但无法在识别中运行那些并得到错误为FailedPreconditionError: FailedPr...onError(). 可能是什么问题呢？

       graph.finalize()
       with tf.variable_scope("Recognition"):
          for i in range(config.train.steps):
             input_data_, input_labels_, input_boxes_ = session.run([input_data, input_labels, input_boxes])
             conv5_3_ = session.run([conv5_3]) #can run this line
             global_step_rec_ = session.run([global_step_rec]) # got FailedPreconditionError: FailedPr...onError() error at this line
             conv7_7_ = session.run([conv7_7])
             h_fmap_ = session.run([h_fmap])

标签： tensorflow

Answer 1

现在它起作用了。由于我的图表有两部分，我需要分别初始化。

(1)首先从预训练模型中获取所有变量，以使用来自检查点的变量进行初始化。然后用 tf.train.Saver 初始化。(2)然后使用初始化其余加载项层tf.global_variables_initializer()

我的代码如下。

       #Initialization
       #Initialize pre-trained model first
       #Since we need to restore pre-trained model and initialize to respective variables in this current graph
       #(1)make a variable list for checkpoint
       #(2)initialize a saver for the variable list
       #(3)then restore
       #(1)
       def print_tensors_in_checkpoint_file(file_name, tensor_name, all_tensors):
          varlist=[]
          reader = pywrap_tensorflow.NewCheckpointReader(file_name)
          if all_tensors:
             var_to_shape_map = reader.get_variable_to_shape_map()
             for key in sorted(var_to_shape_map):
                print(key)
                varlist.append(key)
          return varlist
       varlist=print_tensors_in_checkpoint_file(file_name=config.train.init_checkpoint,all_tensors=True,tensor_name=None)
       #(2)prepare the list of variables by calling variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
       variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
       #countcheckpt_vars=0
       #for n in tf.get_default_graph().as_graph_def().node:
       #   print(n.name)
       #for op in tf.get_default_graph().get_operations():
       #   print(str(op.name))
       #for var in zip(variables):
       #   countcheckpt_vars=countcheckpt_vars+1
       #(3)
       loader = tf.train.Saver(variables[:46])#since I need to initialize only 46 variables from global variables
       tf.logging.info('Initialize from: ' + config.train.init_checkpoint)
       sess_config = tf.ConfigProto(log_device_placement = False, allow_soft_placement = True)
       if FLAGS.gpu_memory_fraction < 0:
          sess_config.gpu_options.allow_growth = True
       elif FLAGS.gpu_memory_fraction > 0:
          sess_config.gpu_options.per_process_gpu_memory_fraction = FLAGS.gpu_memory_fraction;
       session = tf.Session(graph=graph, config=sess_config)
       loader.restore(session, config.train.init_checkpoint)

然后初始化其余变量

init = tf.global_variables_initializer()
session.run(init)