python - 名称“train_data”未定义

Question

我正在尝试通过使用 LSTM 和 seq2seq 建模来实现亚马逊评论的文本摘要模型。但是，这表明 train_data 没有定义。

!pip install pyrouge
import numpy as np
import pandas as pd
import re
from nltk.corpus import stopwords
from google.colab import files

uploaded = files.upload()

for fn in uploaded.keys():
  print('User uploaded file "{name}" with length {length} bytes'.format(
      name=fn, length=len(uploaded[fn])))
from numpy.random import seed
seed(1)

from sklearn.model_selection import train_test_split as tts
import logging

from pyrouge.Rouge155 import Rouge155 
import matplotlib.pyplot as plt
import keras
from keras import backend as k
k.set_learning_phase(1)
from keras import initializers
from keras.optimizers import RMSprop
from keras.models import Model
from keras.layers import Dense,LSTM,Input,Activation,Add,TimeDistributed,\
Permute,Flatten,RepeatVector,merge,Lambda,Multiply,Reshape
from keras.callbacks import ModelCheckpoint
from keras.models import load_model


logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\
    level=logging.INFO)

#######################model params###########################
batch_size = 50
num_classes = 1
epochs = 20
hidden_units = 10
learning_rate = 0.005
clip_norm = 2.0

en_shape=np.shape(train_data["article"][0])
de_shape=np.shape(train_data["summaries"][0])

#######################################################################
############################Helper Functions###########################
#######################################################################    

def encoder_decoder(data):
    print('Encoder_Decoder LSTM...')

    """__encoder___"""
    encoder_inputs = Input(shape=en_shape)

    encoder_LSTM = LSTM(hidden_units,dropout_U=0.2,dropout_W=0.2,return_sequences=True,return_state=True)
    encoder_LSTM_rev=LSTM(hidden_units,return_state=True,return_sequences=True,dropout_U=0.05,dropout_W=0.05,go_backwards=True)

    encoder_outputs, state_h, state_c = encoder_LSTM(encoder_inputs)
    encoder_outputsR, state_hR, state_cR = encoder_LSTM_rev(encoder_inputs)

    state_hfinal=Add()([state_h,state_hR])
    state_cfinal=Add()([state_c,state_cR])
    encoder_outputs_final = Add()([encoder_outputs,encoder_outputsR])

    encoder_states = [state_hfinal,state_cfinal]

    """____decoder___"""
    decoder_inputs = Input(shape=(None,de_shape[1]))
    decoder_LSTM = LSTM(hidden_units,return_sequences=True,dropout_U=0.2,dropout_W=0.2,return_state=True)
    decoder_outputs, _, _ = decoder_LSTM(decoder_inputs,initial_state=encoder_states)

    #Pull out XGBoost, (I mean attention)
    attention = TimeDistributed(Dense(1, activation = 'tanh'))(encoder_outputs_final)
    attention = Flatten()(attention)
    attention = Multiply()([decoder_outputs, attention])
    attention = Activation('softmax')(attention)
    attention = Permute([2, 1])(attention)

    decoder_dense = Dense(de_shape[1],activation='softmax')
    decoder_outputs = decoder_dense(attention)

    model= Model(inputs=[encoder_inputs,decoder_inputs], outputs=decoder_outputs)
    print(model.summary())

    rmsprop = RMSprop(lr=learning_rate,clipnorm=clip_norm)
    model.compile(loss='categorical_crossentropy',optimizer=rmsprop,metrics=['accuracy'])

    x_train,x_test,y_train,y_test=tts(data["article"],data["summaries"],test_size=0.20)
    history= model.fit(x=[x_train,y_train],
              y=y_train,
              batch_size=batch_size,
              epochs=epochs,
              verbose=1,
              validation_data=([x_test,y_test], y_test))
    print(model.summary())
    """_________________inference mode__________________"""
    encoder_model_inf = Model(encoder_inputs,encoder_states)

    decoder_state_input_H = Input(shape=(en_shape[0],))
    decoder_state_input_C = Input(shape=(en_shape[0],)) 
    decoder_state_inputs = [decoder_state_input_H, decoder_state_input_C]
    decoder_outputs, decoder_state_h, decoder_state_c = decoder_LSTM(decoder_inputs,
                                                                     initial_state=decoder_state_inputs)
    decoder_states = [decoder_state_h, decoder_state_c]
    decoder_outputs = decoder_dense(decoder_outputs)

    decoder_model_inf= Model([decoder_inputs]+decoder_state_inputs,
                         [decoder_outputs]+decoder_states)

    scores = model.evaluate([x_test,y_test],y_test, verbose=1)


    print('LSTM test scores:', scores)
    print('\007')
    print(model.summary())
    return model,encoder_model_inf,decoder_model_inf,history


"""_________generate summary from vectors_____________"""


def generateText(SentOfVecs):
    SentOfVecs=np.reshape(SentOfVecs,de_shape)
    kk=""
    for k in SentOfVecs:
        kk = kk + label_encoder.inverse_transform([argmax(k)])[0].strip()+" "
        #kk=kk+((getWord(k)[0]+" ") if getWord(k)[1]>0.01 else "")
    return kk

"""________________generate summary vectors___________"""

def summarize(article):
    stop_pred = False
    article =  np.reshape(article,(1,en_shape[0],en_shape[1]))
    #get initial h and c values from encoder
    init_state_val = encoder.predict(article)
    target_seq = np.zeros((1,1,de_shape[1]))
    #target_seq =np.reshape(train_data['summaries'][k][0],(1,1,de_shape[1]))
    generated_summary=[]
    while not stop_pred:
        decoder_out,decoder_h,decoder_c= decoder.predict(x=[target_seq]+init_state_val)
        generated_summary.append(decoder_out)
        init_state_val= [decoder_h,decoder_c]
        #get most similar word and put in line to be input in next timestep
        #target_seq=np.reshape(model.wv[getWord(decoder_out)[0]],(1,1,emb_size_all))
        target_seq=np.reshape(decoder_out,(1,1,de_shape[1]))
        if len(generated_summary)== de_shape[0]:
            stop_pred=True
            break
    return generated_summary

"""__________________Plot training curves_______________"""

def plot_training(history):
    print(history.history.keys())
    #  "Accuracy"
    plt.plot(history.history['acc'])
    plt.plot(history.history['val_acc'])
    plt.title('model accuracy')
    plt.ylabel('accuracy')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.show()
    # "Loss"
    plt.plot(history.history['loss'])
    plt.plot(history.history['val_loss'])
    plt.title('model loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.show()


def saveModels():
    trained_model.save("%sinit_model"%modelLocation)
    encoder.save("%sencoder"%modelLocation)
    decoder.save("%sdecoder"%modelLocation)

def evaluate_summ(article):
    ref=''
    for k in wt(data['summaries'][article])[:20]:
        ref=ref+' '+k
    gen_sum = generateText(summarize(train_data["article"][article]))
    print("-----------------------------------------------------")
    print("Original summary")
    print(ref)
    print("-----------------------------------------------------")
    print("Generated summary")
    print(gen_sum)
    print("-----------------------------------------------------")
    rouge = Rouge155()
    score = rouge.score_summary(ref, gen_sum)
    print("Rouge1 Score: ",score)

#######################################################################################
################################ Train model and test##################################
#######################################################################################

trained_model,encoder,decoder,history = encoder_decoder(train_data)
plot_training(history)
evaluate_summ(10)

print(generateText(summarize(train_data["article"][8])))
print(data["summaries"][8])
print(data["articles"][78])

它显示以下错误 - NameError: name 'train_data' is not defined。如何准确定义 train_data？

标签： pythonpandasnumpyjupyter-notebook

Answer 1

你需要设置你的训练数据是什么。在定义训练数据是什么之前，请参考 train_data。

训练数据应该从哪里来？您应该从模块中导入它还是从文件中读取它？