python - 我的 model.fit 和 model.evaluate 无法正常工作,并且出现错误
问题描述
train_x = train['text']
valid_x = valid["text"]
train_y = train["label"]
valid_y = valid["label"]
train_y = train_y.values.reshape(-1,1)
vectorizer = TfidfVectorizer()
vectorizer.fit_transform(train_x)
x_train_count = vectorizer.fit_transform(train_x)
x_valid_count = vectorizer.fit_transform(valid_x)
x_test_count = vectorizer.fit_transform(test["text"])
model = tf.keras.Sequential()
model.add(Dense(50,input_dim=x_train_count.shape[1], kernel_initializer="uniform", activation="relu"))
model.add(Dense(1, kernel_initializer="uniform", activation="sigmoid"))
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
# # Fit the model
history = model.fit(x_train_count, train_y, validation_data=(x_valid_count,valid_y), epochs=3, batch_size=128)
loss, acc = model.evaluate(x_test_count, test["label"], verbose=0)
错误
--------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-1-be49478b4c50> in <module>
71 model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
72 # # Fit the model
---> 73 history = model.fit(x_train_count, train_y, validation_data=(x_valid_count,valid_y), epochs=3, batch_size=128)
74
75 loss, acc = model.evaluate(x_test_count, test["label"], verbose=0)
~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1048 training_utils.RespectCompiledTrainableState(self):
1049 # Creates a `tf.data.Dataset` and handles batch and epoch iteration.
-> 1050 data_handler = data_adapter.DataHandler(
1051 x=x,
1052 y=y,
~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution)
1098
1099 adapter_cls = select_data_adapter(x, y)
-> 1100 self._adapter = adapter_cls(
1101 x,
1102 y,
~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\data_adapter.py in __init__(self, x, y, sample_weights, sample_weight_modes, batch_size, steps, shuffle, **kwargs)
564 inputs = pack_x_y_sample_weight(x, y, sample_weights)
565
--> 566 dataset = dataset_ops.DatasetV2.from_tensor_slices(inputs)
567 num_samples = int(nest.flatten(x)[0].shape[0])
568 if shuffle:
~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in from_tensor_slices(tensors)
689 Dataset: A `Dataset`.
690 """
--> 691 return TensorSliceDataset(tensors)
692
693 class _GeneratorState(object):
~\anaconda3\lib\site-packages\tensorflow\python\data\ops\dataset_ops.py in __init__(self, element)
3155 element = structure.normalize_element(element)
3156 batched_spec = structure.type_spec_from_value(element)
-> 3157 self._tensors = structure.to_batched_tensor_list(batched_spec, element)
3158 self._structure = nest.map_structure(
3159 lambda component_spec: component_spec._unbatch(), batched_spec) # pylint: disable=protected-access
~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in to_batched_tensor_list(element_spec, element)
362 # pylint: disable=protected-access
363 # pylint: disable=g-long-lambda
--> 364 return _to_tensor_list_helper(
365 lambda state, spec, component: state + spec._to_batched_tensor_list(
366 component), element_spec, element)
~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in _to_tensor_list_helper(encode_fn, element_spec, element)
337 return encode_fn(state, spec, component)
338
--> 339 return functools.reduce(
340 reduce_fn, zip(nest.flatten(element_spec), nest.flatten(element)), [])
341
~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in reduce_fn(state, value)
335 def reduce_fn(state, value):
336 spec, component = value
--> 337 return encode_fn(state, spec, component)
338
339 return functools.reduce(
~\anaconda3\lib\site-packages\tensorflow\python\data\util\structure.py in <lambda>(state, spec, component)
363 # pylint: disable=g-long-lambda
364 return _to_tensor_list_helper(
--> 365 lambda state, spec, component: state + spec._to_batched_tensor_list(
366 component), element_spec, element)
367
~\anaconda3\lib\site-packages\tensorflow\python\framework\sparse_tensor.py in _to_batched_tensor_list(self, value)
367 raise ValueError(
368 "Unbatching a sparse tensor is only supported for rank >= 1")
--> 369 return [gen_sparse_ops.serialize_many_sparse(
370 value.indices, value.values, value.dense_shape,
371 out_type=dtypes.variant)]
~\anaconda3\lib\site-packages\tensorflow\python\ops\gen_sparse_ops.py in serialize_many_sparse(sparse_indices, sparse_values, sparse_shape, out_type, name)
493 return _result
494 except _core._NotOkStatusException as e:
--> 495 _ops.raise_from_not_ok_status(e, name)
496 except _core._FallbackException:
497 pass
~\anaconda3\lib\site-packages\tensorflow\python\framework\ops.py in raise_from_not_ok_status(e, name)
6860 message = e.message + (" name: " + name if name is not None else "")
6861 # pylint: disable=protected-access
-> 6862 six.raise_from(core._status_to_exception(e.code, message), None)
6863 # pylint: enable=protected-access
6864
~\anaconda3\lib\site-packages\six.py in raise_from(value, from_value)
InvalidArgumentError: indices[1] = [0,40295] 出现故障。许多稀疏操作需要排序索引。用于tf.sparse.reorder
创建正确排序的副本。
解决方案
TfIdfVectorizer 的fit_transform ()和fit()方法返回的对象是压缩的稀疏行格式矩阵。所以为了将它们转换成密集矩阵使用函数。.toarray()
此外,您应该仅在 train_set 上安装 TfIdfVectorizer,然后使用它来转换您的验证和测试集,而无需每次都重新安装,以避免使用来自测试集的数据,因为它可能会引入一些数据泄漏并导致过于乐观的性能. 此外,由于测试和验证集通常很小,仅在它们上拟合 TfIdf 会导致矢量化效果不佳。最后,对于训练、验证和测试集中的相同单词,Idf 因子会有所不同,这通常是不希望的。
出于上述原因,我建议仅在训练集上拟合 TfIdfVectorizer,然后使用拟合的矢量化器来转换验证集和测试集。
这里有一个例子:
vectorizer = TfidfVectorizer()
x_train_count = vectorizer.fit_transform(train_x).toarray()
x_valid_count = vectorizer.transform(valid_x).toarray()
x_test_count = vectorizer.transform(test["text"]).toarray()
如果您的数据不适合内存,您可以考虑一次迭代地转换为密集矩阵一次。这是一个从稀疏 x_train 矩阵创建批处理生成器的函数:
def sparse_matrix_batch_generator(X, y, batch_size=32):
num_samples = np.shape(y)[0]
shuffled_index = np.arange(num_samples)
np.random.shuffle(shuffled_index)
X = X[shuffled_index, :]
y = y[shuffled_index]
for n in range(num_samples//batch_size):
batch_index = shuffled_index[n*batch_size:(n+1)*batch_size]
X_batch = X[batch_index, :].todense()
y_batch = y[batch_index]
yield (X_batch, y_batch)
推荐阅读
- angular - Angular中的核心模块组件和共享模块实现
- cypress - cypress.io - 等待延迟加载的 js 文件
- r - ggplot2中带有“其他”组的堆叠条形图
- php - foreach 迭代以匹配数组内容
- php - 给定两个数组组合公共值
- c# - SqlBulkCopy 是否支持 MsSql 2017 中的图形表?
- mysql - 复制所选数据并增加条目 ID
- git - 为什么 Gitlab 无法从本地网络导入存储库,状态码为 128?
- node.js - mongoose 中的 UpdateMany 无法正常工作,但直接在 mongodb 中可以正常工作
- javascript - 如何解析数组对象,其中每个键在数组中有多个值?