python-3.7 - 如何使用 Gensim word2vec 模型预测未标记数据的情绪？
问题描述
我使用 Gensim word2vec 模型训练和测试了“IMDb 电影评论数据集”，我想预测我自己的未标记数据的情绪。我试过了，但出错了。我正在重用一个开源代码。以下是完整代码：
import pandas as pd
import numpy as np
import text_normalizer as tn
import model_evaluation_utils as meu
np.set_printoptions(precision=2, linewidth=80)
import gensim
import keras
from keras.models import Sequential
from keras.layers import Dropout, Activation, Dense
from sklearn.preprocessing import LabelEncoder
dataset = pd.read_csv(r'imdb_reviews.csv')
new_data = pd.read_csv('abcd.csv', header=0)
# take a peek at the data
print(dataset.head())
reviews = np.array(dataset['reviews'])
sentiments = np.array(dataset['Sentiments'])
# build train and test datasets
train_reviews = reviews[:35000]
train_sentiments = sentiments[:35000]
test_reviews = reviews[35000:]
test_sentiments = sentiments[35000:]
# normalize datasets
norm_train_reviews = tn.normalize_corpus(train_reviews)
norm_test_reviews = tn.normalize_corpus(test_reviews)
le = LabelEncoder()
num_classes=2
# tokenize train reviews & encode train labels
tokenized_train = [tn.tokenizer.tokenize(text)
for text in norm_train_reviews]
y_tr = le.fit_transform(train_sentiments)
y_train = keras.utils.to_categorical(y_tr, num_classes)
# tokenize test reviews & encode test labels
tokenized_test = [tn.tokenizer.tokenize(text)
for text in norm_test_reviews]
y_ts = le.fit_transform(test_sentiments)
y_test = keras.utils.to_categorical(y_ts, num_classes)
# print class label encoding map and encoded labels
print('Sentiment class label map:', dict(zip(le.classes_, le.transform(le.classes_))))
print('Sample test label transformation:\n'+'-'*35,
'\nActual Labels:', test_sentiments[:3], '\nEncoded Labels:', y_ts[:3],
'\nOne hot encoded Labels:\n', y_test[:3])
# build word2vec model
w2v_num_features = 500
w2v_model = gensim.models.Word2Vec(tokenized_train, size=w2v_num_features, window=150,
min_count=10, sample=1e-3)
def averaged_word2vec_vectorizer(corpus, model, num_features):
vocabulary = set(model.wv.index2word)
def average_word_vectors(words, model, vocabulary, num_features):
feature_vector = np.zeros((num_features,), dtype="float64")
nwords = 0.
for word in words:
if word in vocabulary:
nwords = nwords + 1.
feature_vector = np.add(feature_vector, model[word])
if nwords:
feature_vector = np.divide(feature_vector, nwords)
return feature_vector
features = [average_word_vectors(tokenized_sentence, model, vocabulary, num_features)
for tokenized_sentence in corpus]
return np.array(features)
# generate averaged word vector features from word2vec model
avg_wv_train_features = averaged_word2vec_vectorizer(corpus=tokenized_train, model=w2v_model,
num_features=500)
avg_wv_test_features = averaged_word2vec_vectorizer(corpus=tokenized_test, model=w2v_model,
num_features=500)
print('Word2Vec model:> Train features shape:', avg_wv_train_features.shape, ' Test features shape:', avg_wv_test_features
def construct_deepnn_architecture(num_input_features):
dnn_model = Sequential()
dnn_model.add(Dense(512, activation='relu', input_shape=(num_input_features,)))
dnn_model.add(Dropout(0.2))
dnn_model.add(Dense(512, activation='relu'))
dnn_model.add(Dropout(0.2))
dnn_model.add(Dense(512, activation='relu'))
dnn_model.add(Dropout(0.2))
dnn_model.add(Dense(2))
dnn_model.add(Activation('softmax'))
dnn_model.compile(loss='categorical_crossentropy', optimizer='adam',
metrics=['accuracy'])
return dnn_model
w2v_dnn = construct_deepnn_architecture(num_input_features=500)
batch_size = 100
w2v_dnn.fit(avg_wv_train_features, y_train, epochs=15, batch_size=batch_size,
shuffle=True, validation_split=0.1, verbose=1)
y_pred = w2v_dnn.predict_classes(avg_wv_test_features)
predictions = le.inverse_transform(y_pred)
meu.display_model_performance_metrics(true_labels=test_sentiments, predicted_labels=predictions,
classes=['positive', 'negative'])
# This I added to predict and save the results of my own data
pred_y2 = w2v_dnn.predict_classes(new_data['Articles'])
print(pred_y2)
pd.DataFrame(pred_y2, columns=['Sentiments']).to_csv('abcd_sentiments.csv')
当我运行此代码时，出现以下错误：
----> 1 pred_y2 = w2v_dnn.predict_classes(new_data['Articles']) 2 print(pred_y2) 3 pd.DataFrame(pred_y2, columns=['Sentiments']).to_csv 中的 ValueError Traceback（最近一次调用） ('abcd_sentiments.csv')
~/PycharmProjects/News/venv/lib/python3.7/site-packages/keras/engine/sequential.py in predict_classes(self, x, batch_size, verbose) 266 一个 numpy 类预测数组。267 """ --> 268 proba = self.predict(x, batch_size=batch_size, verbose=verbose) 269 if proba.shape[-1] > 1: 270 return proba.argmax(axis=-1)
~/PycharmProjects/News/venv/lib/python3.7/site-packages/keras/engine/training.py in predict(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing) 1439 1440
# Case 2：符号张量或类似 Numpy 数组。-> 1441 x, _, _ = self._standardize_user_data(x) 1442 if self.stateful: 1443 if x[0].shape[0] > batch_size and x[0].shape[0] % batch_size != 0:~/PycharmProjects/News/venv/lib/python3.7/site-packages/keras/engine/training.py in _standardize_user_data(self, x, y, sample_weight, class_weight, check_array_lengths, batch_size) 577 feed_input_shapes, 578 check_batch_axis=False, # 不要强制执行批量大小。--> 579 exception_prefix='input') 580 581 如果 y 不是 None：
~/PycharmProjects/News/venv/lib/python3.7/site-packages/keras/engine/training_utils.py in standardize_input_data（数据，名称，形状，check_batch_axis，exception_prefix）143'：预期'+名称[i]+'有形状 ' + 144 str(shape) + ' 但得到了形状 ' + --> 145 str(data_shape)) 146 返回数据 147
ValueError：检查输入时出错：预期dense_1_input的形状为（500，）但得到的数组形状为（1，）
有人可以建议我如何解决这个错误并预测我未标记数据的情绪吗？我正在使用 Pycharm IDE 中的 python 3.7 和 jupyter notebook。
提前致谢。