############# Buildimg the Sec2Sec model #########
In[37]:
Creating placeholers for the inputs and the targets.
def model_inputs():
inputs = tf.placeholder(tf.int32, [None, None], name = 'input')
targets = tf.placeholder(tf.int32, [None, None], name = 'target')
lr = tf.placeholder(tf.float32, name = 'learning_rate')
keep_prob = tf.placeholder(tf.float32, name = 'keep_prob')
return inputs, targets, lr, keep_prob
In[38]:
Preprocessing the targets.
def preprocess_targets(targets, word2int, batch_size):
left_side = tf.fill([batch_size, 1], word2int[''])
right_side = tf.strided_slice(targets, [0,0], [batch_size, -1], [1,1])
preprocessed_targets = tf.concat([left_side, right_side], 1)
return preprocessed_targets
In[39]:
Creating the Encoder RNN Layer.
def encoder_rnn(rnn_inputs, rnn_size, num_layers, keep_prob, sequence_length):
lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)
lstm_dropout = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)
encoder_cell = tf.contrib.rnn.MultiRNNCell([lstm_dropout] * num_layers)
encoder_output, encoder_state = tf.nn.bidirectional_dynamic_rnn(cell_fw = encoder_cell,
cell_bw = encoder_cell,
sequence_length = sequence_length,
inputs = rnn_inputs,
dtype = tf.float32)
In[40]:
Decoding the Training set.
def decode_training_set(encoder_state, decoder_cell, decoder_embedded_input, sequence_length, decoding_scope, output_function, keep_prob, batch_size):
attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size])
attention_keys, attention_values, attention_score_function, attention_construct_function = tf.contrib.seq2seq.prepare_attention(attention_states, attention_option = "bahdanau", num_units = decoder_cell.output_size)
training_decoder_function = tf.contrib.seq2seq.attention_decoder_fn_train(encoder_state[0],
attention_keys,
attention_values,
attention_score_function,
attention_construct_function,
name = "attn_dec_train")
decoder_output, decoder_final_state, decoder_final_context_state = tf.contrib.seq2seq.dynamic_rnn_decoder(decoder_cell,
training_decoder_function,
decoder_embedded_input,
sequence_length,
scope = decoding_scope)
decoder_output_dropout = tf.nn.dropout(decoder_output, keep_prob)
return output_function(decoder_output_dropout)
In[41]:
Decoding the Test / Validation Set.
def decode_test_set(encoder_state, decoder_cell, decoder_embeddings_matrix, sos_id, eos_id, maximum_length, num_words, decoding_scope, output_function, keep_prob, batch_size):
attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size])
attention_keys, attention_values, attention_score_function, attention_construct_function = tf.contrib.seq2seq.prepare_attention(attention_states, attention_option = "bahdanau", num_units = decoder_cell.output_size)
test_decoder_function = tf.contrib.seq2seq.attention_decoder_fn_inference(output_function,
encoder_state[0],
attention_keys,
attention_values,
attention_score_function,
attention_construct_function,
decoder_embeddings_matrix,
sos_id,
eos_id,
maximum_length,
num_words,
name = "attn_dec_inf")
test_predictions, decoder_final_state, decoder_final_context_state = tf.contrib.seq2seq.dynamic_rnn_decoder(decoder_cell,
test_decoder_function,
scope = decoding_scope)
return test_predictions
In[42]:
Creating the Decoder RNN
def decoder_rnn(decoder_embedded_input, decoder_embeddings_matrix, encoder_state, num_words, sequence_length, rnn_size, num_layers, word2int, keep_prob, batch_size):
with tf.variable_scope("decoding") as decoding_scope:
lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)
lstm_dropout = tf.contrib.rnn.DropoutWrapper(lstm, input_keep_prob = keep_prob)
decoder_cell = tf.contrib.rnn.MultiRNNCell([lstm_dropout] * num_layers)
weights = tf.truncated_normal_initializer(stddev = 0.1)
biases = tf.zeros_initializer()
output_function = lambda x: tf.contrib.layers.fully_connected(x,
num_words,
None,
scope = decoding_scope,
weights_initializer = weights,
biases_initializer = biases)
training_predictions = decode_training_set(encoder_state,
decoder_cell,
decoder_embedded_input,
sequence_length,
decoding_scope,
output_function,
keep_prob,
batch_size)
decoding_scope.reuse_variables()
test_predictions = decode_test_set(encoder_state,
decoder_cell,
decoder_embeddings_matrix,
word2int[''],
word2int[''],
sequence_length - 1,
num_words,
decoding_scope,
output_function,
keep_prob,
batch_size)
return training_predictions, test_predictions
In[43]:
Building the Sec2Sec model.
def seq2seq_model(inputs, targets, keep_prob, batch_size, sequence_length, answers_num_words, questions_num_words, encoder_embedding_size, decoder_embedding_size, rnn_size, num_layers, questionswords2int):
encoder_embedded_input = tf.contrib.layers.embed_sequence(inputs,
answers_num_words + 1,
encoder_embedding_size,
initializer = tf.random_uniform_initializer(0, 1))
encoder_state = encoder_rnn(encoder_embedded_input, rnn_size, num_layers, keep_prob, sequence_length)
preprocessed_targets = preprocess_targets(targets, questionswords2int, batch_size)
decoder_embeddings_matrix = tf.Variable(tf.random_uniform([questions_num_words + 1, decoder_embedding_size], 0, 1))
decoder_embedded_input = tf.nn.embedding_lookup(decoder_embeddings_matrix, preprocessed_targets)
training_predictions, test_predictions = decoder_rnn(decoder_embedded_input,
decoder_embeddings_matrix,
encoder_state,
questions_num_words,
sequence_length,
rnn_size,
num_layers,
questionswords2int,
keep_prob,
batch_size)
return training_predictions, test_predictions
In[44]:
Setting the Hyperparameters.
epochs = 100
batch_size = 64
rnn_size = 512
num_layers = 3
encoding_embedding_size = 512
decoding_embedding_size = 512
learning_rate = 0.01
learning_name_decay = 0.9
min_learning_rate = 0.0001
keep_probability = 0.5
In[45]:
Defining a session
tf.reset_default_graph()
session = tf.InteractiveSession()
In[46]:
Loading the model inputs
inputs, targets, lr, keep_prob = model_inputs()
In[47]:
Setting the sequence_length
sequence_length = tf.placeholder_with_default(25, None, name = 'sequence_length')
In[48]:
Getting the shape of the inputs tensor.
input_shape = tf.shape(inputs)
In[49]:
Getting the training and test predictions.
training_predictions, test_predictions = seq2seq_model(tf.reverse(inputs, [-1]),
targets,
keep_prob,
batch_size,
sequence_length,
len(answerswords2int),
len(questionswords2int),
encoding_embedding_size,
decoding_embedding_size,
rnn_size,
num_layers,
questionswords2int)
TypeError Traceback (most recent call last)
in ()
12 rnn_size,
13 num_layers,
---> 14 questionswords2int)
in seq2seq_model(inputs, targets, keep_prob, batch_size, sequence_length, answers_num_words, questions_num_words, encoder_embedding_size, decoder_embedding_size, rnn_size, num_layers, questionswords2int)
19 questionswords2int,
20 keep_prob,
---> 21 batch_size)
22 return training_predictions, test_predictions
in decoder_rnn(decoder_embedded_input, decoder_embeddings_matrix, encoder_state, num_words, sequence_length, rnn_size, num_layers, word2int, keep_prob, batch_size)
21 output_function,
22 keep_prob,
---> 23 batch_size)
24 decoding_scope.reuse_variables()
25 test_predictions = decode_test_set(encoder_state,
in decode_training_set(encoder_state, decoder_cell, decoder_embedded_input, sequence_length, decoding_scope, output_function, keep_prob, batch_size)
4 attention_states = tf.zeros([batch_size, 1, decoder_cell.output_size])
5 attention_keys, attention_values, attention_score_function, attention_construct_function = tf.contrib.seq2seq.prepare_attention(attention_states, attention_option = "bahdanau", num_units = decoder_cell.output_size)
----> 6 training_decoder_function = tf.contrib.seq2seq.attention_decoder_fn_train(encoder_state[0],
7 attention_keys,
8 attention_values,
TypeError: 'NoneType' object is not subscriptable
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent