Skip to content

GitLab

Commit 124a501d authored by Quoc Le's avatar Quoc Le
Browse files

fix the readme

parent 33c4e784
Hide whitespace changes
Inline Side-by-side
Showing with 2282 additions and 0 deletions
README.md
View file @ 124a501d
...@@ -20,3 +20,22 @@ To propose a model for inclusion please submit a pull request. ...@@ -20,3 +20,22 @@ To propose a model for inclusion please submit a pull request.
- [textsum](textsum) -- sequence-to-sequence with attention model for text summarization. - [textsum](textsum) -- sequence-to-sequence with attention model for text summarization.
- [transformer](transformer) -- spatial transformer network, which allows the spatial manipulation of data within the network - [transformer](transformer) -- spatial transformer network, which allows the spatial manipulation of data within the network
- [im2txt](im2txt) -- image-to-text neural network for image captioning. - [im2txt](im2txt) -- image-to-text neural network for image captioning.
=======
Implementation of the Neural Programmer model described in https://openreview.net/pdf?id=ry2YOrcge
Download the data from http://www-nlp.stanford.edu/software/sempre/wikitable/
Change the data_dir FLAG to the location of the data
Training:
python neural_programmer.py
The models are written to FLAGS.output_dir
Testing:
python neural_programmer.py --evaluator_job=True
The models are loaded from FLAGS.output_dir.
The evaluation is done on development data.
Maintained by Arvind Neelakantan (arvind2505)
data_utils.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Functions for constructing vocabulary, converting the examples to integer format and building the required masks for batch computation Author: aneelakantan (Arvind Neelakantan)
"""
import copy
import numbers
import numpy as np
import wiki_data
def return_index(a):
for i in range(len(a)):
if (a[i] == 1.0):
return i
def construct_vocab(data, utility, add_word=False):
ans = []
for example in data:
sent = ""
for word in example.question:
if (not (isinstance(word, numbers.Number))):
sent += word + " "
example.original_nc = copy.deepcopy(example.number_columns)
example.original_wc = copy.deepcopy(example.word_columns)
example.original_nc_names = copy.deepcopy(example.number_column_names)
example.original_wc_names = copy.deepcopy(example.word_column_names)
if (add_word):
continue
number_found = 0
if (not (example.is_bad_example)):
for word in example.question:
if (isinstance(word, numbers.Number)):
number_found += 1
else:
if (not (utility.word_ids.has_key(word))):
utility.words.append(word)
utility.word_count[word] = 1
utility.word_ids[word] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[word]] = word
else:
utility.word_count[word] += 1
for col_name in example.word_column_names:
for word in col_name:
if (isinstance(word, numbers.Number)):
number_found += 1
else:
if (not (utility.word_ids.has_key(word))):
utility.words.append(word)
utility.word_count[word] = 1
utility.word_ids[word] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[word]] = word
else:
utility.word_count[word] += 1
for col_name in example.number_column_names:
for word in col_name:
if (isinstance(word, numbers.Number)):
number_found += 1
else:
if (not (utility.word_ids.has_key(word))):
utility.words.append(word)
utility.word_count[word] = 1
utility.word_ids[word] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[word]] = word
else:
utility.word_count[word] += 1
def word_lookup(word, utility):
if (utility.word_ids.has_key(word)):
return word
else:
return utility.unk_token
def convert_to_int_2d_and_pad(a, utility):
ans = []
#print a
for b in a:
temp = []
if (len(b) > utility.FLAGS.max_entry_length):
b = b[0:utility.FLAGS.max_entry_length]
for remaining in range(len(b), utility.FLAGS.max_entry_length):
b.append(utility.dummy_token)
assert len(b) == utility.FLAGS.max_entry_length
for word in b:
temp.append(utility.word_ids[word_lookup(word, utility)])
ans.append(temp)
#print ans
return ans
def convert_to_bool_and_pad(a, utility):
a = a.tolist()
for i in range(len(a)):
for j in range(len(a[i])):
if (a[i][j] < 1):
a[i][j] = False
else:
a[i][j] = True
a[i] = a[i] + [False] * (utility.FLAGS.max_elements - len(a[i]))
return a
seen_tables = {}
def partial_match(question, table, number):
answer = []
match = {}
for i in range(len(table)):
temp = []
for j in range(len(table[i])):
temp.append(0)
answer.append(temp)
for i in range(len(table)):
for j in range(len(table[i])):
for word in question:
if (number):
if (word == table[i][j]):
answer[i][j] = 1.0
match[i] = 1.0
else:
if (word in table[i][j]):
answer[i][j] = 1.0
match[i] = 1.0
return answer, match
def exact_match(question, table, number):
#performs exact match operation
answer = []
match = {}
matched_indices = []
for i in range(len(table)):
temp = []
for j in range(len(table[i])):
temp.append(0)
answer.append(temp)
for i in range(len(table)):
for j in range(len(table[i])):
if (number):
for word in question:
if (word == table[i][j]):
match[i] = 1.0
answer[i][j] = 1.0
else:
table_entry = table[i][j]
for k in range(len(question)):
if (k + len(table_entry) <= len(question)):
if (table_entry == question[k:(k + len(table_entry))]):
#if(len(table_entry) == 1):
#print "match: ", table_entry, question
match[i] = 1.0
answer[i][j] = 1.0
matched_indices.append((k, len(table_entry)))
return answer, match, matched_indices
def partial_column_match(question, table, number):
answer = []
for i in range(len(table)):
answer.append(0)
for i in range(len(table)):
for word in question:
if (word in table[i]):
answer[i] = 1.0
return answer
def exact_column_match(question, table, number):
#performs exact match on column names
answer = []
matched_indices = []
for i in range(len(table)):
answer.append(0)
for i in range(len(table)):
table_entry = table[i]
for k in range(len(question)):
if (k + len(table_entry) <= len(question)):
if (table_entry == question[k:(k + len(table_entry))]):
answer[i] = 1.0
matched_indices.append((k, len(table_entry)))
return answer, matched_indices
def get_max_entry(a):
e = {}
for w in a:
if (w != "UNK, "):
if (e.has_key(w)):
e[w] += 1
else:
e[w] = 1
if (len(e) > 0):
(key, val) = sorted(e.items(), key=lambda x: -1 * x[1])[0]
if (val > 1):
return key
else:
return -1.0
else:
return -1.0
def list_join(a):
ans = ""
for w in a:
ans += str(w) + ", "
return ans
def group_by_max(table, number):
#computes the most frequently occuring entry in a column
answer = []
for i in range(len(table)):
temp = []
for j in range(len(table[i])):
temp.append(0)
answer.append(temp)
for i in range(len(table)):
if (number):
curr = table[i]
else:
curr = [list_join(w) for w in table[i]]
max_entry = get_max_entry(curr)
#print i, max_entry
for j in range(len(curr)):
if (max_entry == curr[j]):
answer[i][j] = 1.0
else:
answer[i][j] = 0.0
return answer
def pick_one(a):
for i in range(len(a)):
if (1.0 in a[i]):
return True
return False
def check_processed_cols(col, utility):
return True in [
True for y in col
if (y != utility.FLAGS.pad_int and y !=
utility.FLAGS.bad_number_pre_process)
]
def complete_wiki_processing(data, utility, train=True):
#convert to integers and padding
processed_data = []
num_bad_examples = 0
for example in data:
number_found = 0
if (example.is_bad_example):
num_bad_examples += 1
if (not (example.is_bad_example)):
example.string_question = example.question[:]
#entry match
example.processed_number_columns = example.processed_number_columns[:]
example.processed_word_columns = example.processed_word_columns[:]
example.word_exact_match, word_match, matched_indices = exact_match(
example.string_question, example.original_wc, number=False)
example.number_exact_match, number_match, _ = exact_match(
example.string_question, example.original_nc, number=True)
if (not (pick_one(example.word_exact_match)) and not (
pick_one(example.number_exact_match))):
assert len(word_match) == 0
assert len(number_match) == 0
example.word_exact_match, word_match = partial_match(
example.string_question, example.original_wc, number=False)
#group by max
example.word_group_by_max = group_by_max(example.original_wc, False)
example.number_group_by_max = group_by_max(example.original_nc, True)
#column name match
example.word_column_exact_match, wcol_matched_indices = exact_column_match(
example.string_question, example.original_wc_names, number=False)
example.number_column_exact_match, ncol_matched_indices = exact_column_match(
example.string_question, example.original_nc_names, number=False)
if (not (1.0 in example.word_column_exact_match) and not (
1.0 in example.number_column_exact_match)):
example.word_column_exact_match = partial_column_match(
example.string_question, example.original_wc_names, number=False)
example.number_column_exact_match = partial_column_match(
example.string_question, example.original_nc_names, number=False)
if (len(word_match) > 0 or len(number_match) > 0):
example.question.append(utility.entry_match_token)
if (1.0 in example.word_column_exact_match or
1.0 in example.number_column_exact_match):
example.question.append(utility.column_match_token)
example.string_question = example.question[:]
example.number_lookup_matrix = np.transpose(
example.number_lookup_matrix)[:]
example.word_lookup_matrix = np.transpose(example.word_lookup_matrix)[:]
example.columns = example.number_columns[:]
example.word_columns = example.word_columns[:]
example.len_total_cols = len(example.word_column_names) + len(
example.number_column_names)
example.column_names = example.number_column_names[:]
example.word_column_names = example.word_column_names[:]
example.string_column_names = example.number_column_names[:]
example.string_word_column_names = example.word_column_names[:]
example.sorted_number_index = []
example.sorted_word_index = []
example.column_mask = []
example.word_column_mask = []
example.processed_column_mask = []
example.processed_word_column_mask = []
example.word_column_entry_mask = []
example.question_attention_mask = []
example.question_number = example.question_number_1 = -1
example.question_attention_mask = []
example.ordinal_question = []
example.ordinal_question_one = []
new_question = []
if (len(example.number_columns) > 0):
example.len_col = len(example.number_columns[0])
else:
example.len_col = len(example.word_columns[0])
for (start, length) in matched_indices:
for j in range(length):
example.question[start + j] = utility.unk_token
#print example.question
for word in example.question:
if (isinstance(word, numbers.Number) or wiki_data.is_date(word)):
if (not (isinstance(word, numbers.Number)) and
wiki_data.is_date(word)):
word = word.replace("X", "").replace("-", "")
number_found += 1
if (number_found == 1):
example.question_number = word
if (len(example.ordinal_question) > 0):
example.ordinal_question[len(example.ordinal_question) - 1] = 1.0
else:
example.ordinal_question.append(1.0)
elif (number_found == 2):
example.question_number_1 = word
if (len(example.ordinal_question_one) > 0):
example.ordinal_question_one[len(example.ordinal_question_one) -
1] = 1.0
else:
example.ordinal_question_one.append(1.0)
else:
new_question.append(word)
example.ordinal_question.append(0.0)
example.ordinal_question_one.append(0.0)
example.question = [
utility.word_ids[word_lookup(w, utility)] for w in new_question
]
example.question_attention_mask = [0.0] * len(example.question)
#when the first question number occurs before a word
example.ordinal_question = example.ordinal_question[0:len(
example.question)]
example.ordinal_question_one = example.ordinal_question_one[0:len(
example.question)]
#question-padding
example.question = [utility.word_ids[utility.dummy_token]] * (
utility.FLAGS.question_length - len(example.question)
) + example.question
example.question_attention_mask = [-10000.0] * (
utility.FLAGS.question_length - len(example.question_attention_mask)
) + example.question_attention_mask
example.ordinal_question = [0.0] * (utility.FLAGS.question_length -
len(example.ordinal_question)
) + example.ordinal_question
example.ordinal_question_one = [0.0] * (utility.FLAGS.question_length -
len(example.ordinal_question_one)
) + example.ordinal_question_one
if (True):
#number columns and related-padding
num_cols = len(example.columns)
start = 0
for column in example.number_columns:
if (check_processed_cols(example.processed_number_columns[start],
utility)):
example.processed_column_mask.append(0.0)
sorted_index = sorted(
range(len(example.processed_number_columns[start])),
key=lambda k: example.processed_number_columns[start][k],
reverse=True)
sorted_index = sorted_index + [utility.FLAGS.pad_int] * (
utility.FLAGS.max_elements - len(sorted_index))
example.sorted_number_index.append(sorted_index)
example.columns[start] = column + [utility.FLAGS.pad_int] * (
utility.FLAGS.max_elements - len(column))
example.processed_number_columns[start] += [utility.FLAGS.pad_int] * (
utility.FLAGS.max_elements -
len(example.processed_number_columns[start]))
start += 1
example.column_mask.append(0.0)
for remaining in range(num_cols, utility.FLAGS.max_number_cols):
example.sorted_number_index.append([utility.FLAGS.pad_int] *
(utility.FLAGS.max_elements))
example.columns.append([utility.FLAGS.pad_int] *
(utility.FLAGS.max_elements))
example.processed_number_columns.append([utility.FLAGS.pad_int] *
(utility.FLAGS.max_elements))
example.number_exact_match.append([0.0] *
(utility.FLAGS.max_elements))
example.number_group_by_max.append([0.0] *
(utility.FLAGS.max_elements))
example.column_mask.append(-100000000.0)
example.processed_column_mask.append(-100000000.0)
example.number_column_exact_match.append(0.0)
example.column_names.append([utility.dummy_token])
#word column and related-padding
start = 0
word_num_cols = len(example.word_columns)
for column in example.word_columns:
if (check_processed_cols(example.processed_word_columns[start],
utility)):
example.processed_word_column_mask.append(0.0)
sorted_index = sorted(
range(len(example.processed_word_columns[start])),
key=lambda k: example.processed_word_columns[start][k],
reverse=True)
sorted_index = sorted_index + [utility.FLAGS.pad_int] * (
utility.FLAGS.max_elements - len(sorted_index))
example.sorted_word_index.append(sorted_index)
column = convert_to_int_2d_and_pad(column, utility)
example.word_columns[start] = column + [[
utility.word_ids[utility.dummy_token]
] * utility.FLAGS.max_entry_length] * (utility.FLAGS.max_elements -
len(column))
example.processed_word_columns[start] += [utility.FLAGS.pad_int] * (
utility.FLAGS.max_elements -
len(example.processed_word_columns[start]))
example.word_column_entry_mask.append([0] * len(column) + [
utility.word_ids[utility.dummy_token]
] * (utility.FLAGS.max_elements - len(column)))
start += 1
example.word_column_mask.append(0.0)
for remaining in range(word_num_cols, utility.FLAGS.max_word_cols):
example.sorted_word_index.append([utility.FLAGS.pad_int] *
(utility.FLAGS.max_elements))
example.word_columns.append([[utility.word_ids[utility.dummy_token]] *
utility.FLAGS.max_entry_length] *
(utility.FLAGS.max_elements))
example.word_column_entry_mask.append(
[utility.word_ids[utility.dummy_token]] *
(utility.FLAGS.max_elements))
example.word_exact_match.append([0.0] * (utility.FLAGS.max_elements))
example.word_group_by_max.append([0.0] * (utility.FLAGS.max_elements))
example.processed_word_columns.append([utility.FLAGS.pad_int] *
(utility.FLAGS.max_elements))
example.word_column_mask.append(-100000000.0)
example.processed_word_column_mask.append(-100000000.0)
example.word_column_exact_match.append(0.0)
example.word_column_names.append([utility.dummy_token] *
utility.FLAGS.max_entry_length)
seen_tables[example.table_key] = 1
#convert column and word column names to integers
example.column_ids = convert_to_int_2d_and_pad(example.column_names,
utility)
example.word_column_ids = convert_to_int_2d_and_pad(
example.word_column_names, utility)
for i_em in range(len(example.number_exact_match)):
example.number_exact_match[i_em] = example.number_exact_match[
i_em] + [0.0] * (utility.FLAGS.max_elements -
len(example.number_exact_match[i_em]))
example.number_group_by_max[i_em] = example.number_group_by_max[
i_em] + [0.0] * (utility.FLAGS.max_elements -
len(example.number_group_by_max[i_em]))
for i_em in range(len(example.word_exact_match)):
example.word_exact_match[i_em] = example.word_exact_match[
i_em] + [0.0] * (utility.FLAGS.max_elements -
len(example.word_exact_match[i_em]))
example.word_group_by_max[i_em] = example.word_group_by_max[
i_em] + [0.0] * (utility.FLAGS.max_elements -
len(example.word_group_by_max[i_em]))
example.exact_match = example.number_exact_match + example.word_exact_match
example.group_by_max = example.number_group_by_max + example.word_group_by_max
example.exact_column_match = example.number_column_exact_match + example.word_column_exact_match
#answer and related mask, padding
if (example.is_lookup):
example.answer = example.calc_answer
example.number_print_answer = example.number_lookup_matrix.tolist()
example.word_print_answer = example.word_lookup_matrix.tolist()
for i_answer in range(len(example.number_print_answer)):
example.number_print_answer[i_answer] = example.number_print_answer[
i_answer] + [0.0] * (utility.FLAGS.max_elements -
len(example.number_print_answer[i_answer]))
for i_answer in range(len(example.word_print_answer)):
example.word_print_answer[i_answer] = example.word_print_answer[
i_answer] + [0.0] * (utility.FLAGS.max_elements -
len(example.word_print_answer[i_answer]))
example.number_lookup_matrix = convert_to_bool_and_pad(
example.number_lookup_matrix, utility)
example.word_lookup_matrix = convert_to_bool_and_pad(
example.word_lookup_matrix, utility)
for remaining in range(num_cols, utility.FLAGS.max_number_cols):
example.number_lookup_matrix.append([False] *
utility.FLAGS.max_elements)
example.number_print_answer.append([0.0] * utility.FLAGS.max_elements)
for remaining in range(word_num_cols, utility.FLAGS.max_word_cols):
example.word_lookup_matrix.append([False] *
utility.FLAGS.max_elements)
example.word_print_answer.append([0.0] * utility.FLAGS.max_elements)
example.print_answer = example.number_print_answer + example.word_print_answer
else:
example.answer = example.calc_answer
example.print_answer = [[0.0] * (utility.FLAGS.max_elements)] * (
utility.FLAGS.max_number_cols + utility.FLAGS.max_word_cols)
#question_number masks
if (example.question_number == -1):
example.question_number_mask = np.zeros([utility.FLAGS.max_elements])
else:
example.question_number_mask = np.ones([utility.FLAGS.max_elements])
if (example.question_number_1 == -1):
example.question_number_one_mask = -10000.0
else:
example.question_number_one_mask = np.float64(0.0)
if (example.len_col > utility.FLAGS.max_elements):
continue
processed_data.append(example)
return processed_data
def add_special_words(utility):
utility.words.append(utility.entry_match_token)
utility.word_ids[utility.entry_match_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[
utility.entry_match_token]] = utility.entry_match_token
utility.entry_match_token_id = utility.word_ids[utility.entry_match_token]
print "entry match token: ", utility.word_ids[
utility.entry_match_token], utility.entry_match_token_id
utility.words.append(utility.column_match_token)
utility.word_ids[utility.column_match_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[
utility.column_match_token]] = utility.column_match_token
utility.column_match_token_id = utility.word_ids[utility.column_match_token]
print "entry match token: ", utility.word_ids[
utility.column_match_token], utility.column_match_token_id
utility.words.append(utility.dummy_token)
utility.word_ids[utility.dummy_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[
utility.dummy_token]] = utility.dummy_token
utility.dummy_token_id = utility.word_ids[utility.dummy_token]
utility.words.append(utility.unk_token)
utility.word_ids[utility.unk_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[
utility.unk_token]] = utility.unk_token
def perform_word_cutoff(utility):
if (utility.FLAGS.word_cutoff > 0):
for word in utility.word_ids.keys():
if (utility.word_count.has_key(word) and utility.word_count[word] <
utility.FLAGS.word_cutoff and word != utility.unk_token and
word != utility.dummy_token and word != utility.entry_match_token and
word != utility.column_match_token):
utility.word_ids.pop(word)
utility.words.remove(word)
def word_dropout(question, utility):
if (utility.FLAGS.word_dropout_prob > 0.0):
new_question = []
for i in range(len(question)):
if (question[i] != utility.dummy_token_id and
utility.random.random() > utility.FLAGS.word_dropout_prob):
new_question.append(utility.word_ids[utility.unk_token])
else:
new_question.append(question[i])
return new_question
else:
return question
def generate_feed_dict(data, curr, batch_size, gr, train=False, utility=None):
#prepare feed dict dictionary
feed_dict = {}
feed_examples = []
for j in range(batch_size):
feed_examples.append(data[curr + j])
if (train):
feed_dict[gr.batch_question] = [
word_dropout(feed_examples[j].question, utility)
for j in range(batch_size)
]
else:
feed_dict[gr.batch_question] = [
feed_examples[j].question for j in range(batch_size)
]
feed_dict[gr.batch_question_attention_mask] = [
feed_examples[j].question_attention_mask for j in range(batch_size)
]
feed_dict[
gr.batch_answer] = [feed_examples[j].answer for j in range(batch_size)]
feed_dict[gr.batch_number_column] = [
feed_examples[j].columns for j in range(batch_size)
]
feed_dict[gr.batch_processed_number_column] = [
feed_examples[j].processed_number_columns for j in range(batch_size)
]
feed_dict[gr.batch_processed_sorted_index_number_column] = [
feed_examples[j].sorted_number_index for j in range(batch_size)
]
feed_dict[gr.batch_processed_sorted_index_word_column] = [
feed_examples[j].sorted_word_index for j in range(batch_size)
]
feed_dict[gr.batch_question_number] = np.array(
[feed_examples[j].question_number for j in range(batch_size)]).reshape(
(batch_size, 1))
feed_dict[gr.batch_question_number_one] = np.array(
[feed_examples[j].question_number_1 for j in range(batch_size)]).reshape(
(batch_size, 1))
feed_dict[gr.batch_question_number_mask] = [
feed_examples[j].question_number_mask for j in range(batch_size)
]
feed_dict[gr.batch_question_number_one_mask] = np.array(
[feed_examples[j].question_number_one_mask for j in range(batch_size)
]).reshape((batch_size, 1))
feed_dict[gr.batch_print_answer] = [
feed_examples[j].print_answer for j in range(batch_size)
]
feed_dict[gr.batch_exact_match] = [
feed_examples[j].exact_match for j in range(batch_size)
]
feed_dict[gr.batch_group_by_max] = [
feed_examples[j].group_by_max for j in range(batch_size)
]
feed_dict[gr.batch_column_exact_match] = [
feed_examples[j].exact_column_match for j in range(batch_size)
]
feed_dict[gr.batch_ordinal_question] = [
feed_examples[j].ordinal_question for j in range(batch_size)
]
feed_dict[gr.batch_ordinal_question_one] = [
feed_examples[j].ordinal_question_one for j in range(batch_size)
]
feed_dict[gr.batch_number_column_mask] = [
feed_examples[j].column_mask for j in range(batch_size)
]
feed_dict[gr.batch_number_column_names] = [
feed_examples[j].column_ids for j in range(batch_size)
]
feed_dict[gr.batch_processed_word_column] = [
feed_examples[j].processed_word_columns for j in range(batch_size)
]
feed_dict[gr.batch_word_column_mask] = [
feed_examples[j].word_column_mask for j in range(batch_size)
]
feed_dict[gr.batch_word_column_names] = [
feed_examples[j].word_column_ids for j in range(batch_size)
]
feed_dict[gr.batch_word_column_entry_mask] = [
feed_examples[j].word_column_entry_mask for j in range(batch_size)
]
return feed_dict
model.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Author: aneelakantan (Arvind Neelakantan)
"""
import numpy as np
import tensorflow as tf
import nn_utils
class Graph():
def __init__(self, utility, batch_size, max_passes, mode="train"):
self.utility = utility
self.data_type = self.utility.tf_data_type[self.utility.FLAGS.data_type]
self.max_elements = self.utility.FLAGS.max_elements
max_elements = self.utility.FLAGS.max_elements
self.num_cols = self.utility.FLAGS.max_number_cols
self.num_word_cols = self.utility.FLAGS.max_word_cols
self.question_length = self.utility.FLAGS.question_length
self.batch_size = batch_size
self.max_passes = max_passes
self.mode = mode
self.embedding_dims = self.utility.FLAGS.embedding_dims
#input question and a mask
self.batch_question = tf.placeholder(tf.int32,
[batch_size, self.question_length])
self.batch_question_attention_mask = tf.placeholder(
self.data_type, [batch_size, self.question_length])
#ground truth scalar answer and lookup answer
self.batch_answer = tf.placeholder(self.data_type, [batch_size])
self.batch_print_answer = tf.placeholder(
self.data_type,
[batch_size, self.num_cols + self.num_word_cols, max_elements])
#number columns and its processed version
self.batch_number_column = tf.placeholder(
self.data_type, [batch_size, self.num_cols, max_elements
]) #columns with numeric entries
self.batch_processed_number_column = tf.placeholder(
self.data_type, [batch_size, self.num_cols, max_elements])
self.batch_processed_sorted_index_number_column = tf.placeholder(
tf.int32, [batch_size, self.num_cols, max_elements])
#word columns and its processed version
self.batch_processed_word_column = tf.placeholder(
self.data_type, [batch_size, self.num_word_cols, max_elements])
self.batch_processed_sorted_index_word_column = tf.placeholder(
tf.int32, [batch_size, self.num_word_cols, max_elements])
self.batch_word_column_entry_mask = tf.placeholder(
tf.int32, [batch_size, self.num_word_cols, max_elements])
#names of word and number columns along with their mask
self.batch_word_column_names = tf.placeholder(
tf.int32,
[batch_size, self.num_word_cols, self.utility.FLAGS.max_entry_length])
self.batch_word_column_mask = tf.placeholder(
self.data_type, [batch_size, self.num_word_cols])
self.batch_number_column_names = tf.placeholder(
tf.int32,
[batch_size, self.num_cols, self.utility.FLAGS.max_entry_length])
self.batch_number_column_mask = tf.placeholder(self.data_type,
[batch_size, self.num_cols])
#exact match and group by max operation
self.batch_exact_match = tf.placeholder(
self.data_type,
[batch_size, self.num_cols + self.num_word_cols, max_elements])
self.batch_column_exact_match = tf.placeholder(
self.data_type, [batch_size, self.num_cols + self.num_word_cols])
self.batch_group_by_max = tf.placeholder(
self.data_type,
[batch_size, self.num_cols + self.num_word_cols, max_elements])
#numbers in the question along with their position. This is used to compute arguments to the comparison operations
self.batch_question_number = tf.placeholder(self.data_type, [batch_size, 1])
self.batch_question_number_one = tf.placeholder(self.data_type,
[batch_size, 1])
self.batch_question_number_mask = tf.placeholder(
self.data_type, [batch_size, max_elements])
self.batch_question_number_one_mask = tf.placeholder(self.data_type,
[batch_size, 1])
self.batch_ordinal_question = tf.placeholder(
self.data_type, [batch_size, self.question_length])
self.batch_ordinal_question_one = tf.placeholder(
self.data_type, [batch_size, self.question_length])
def LSTM_question_embedding(self, sentence, sentence_length):
#LSTM processes the input question
lstm_params = "question_lstm"
hidden_vectors = []
sentence = self.batch_question
question_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
question_c_hidden = tf.zeros(
[self.batch_size, self.utility.FLAGS.embedding_dims], self.data_type)
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
rnn_dropout_mask = tf.cast(
tf.random_uniform(
tf.shape(question_hidden), minval=0.0, maxval=1.0) <
self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
rnn_dropout_mask = tf.ones_like(question_hidden)
for question_iterator in range(self.question_length):
curr_word = sentence[:, question_iterator]
question_vector = nn_utils.apply_dropout(
nn_utils.get_embedding(curr_word, self.utility, self.params),
self.utility.FLAGS.dropout, self.mode)
question_hidden, question_c_hidden = nn_utils.LSTMCell(
question_vector, question_hidden, question_c_hidden, lstm_params,
self.params)
if (self.utility.FLAGS.rnn_dropout > 0.0):
question_hidden = question_hidden * rnn_dropout_mask
hidden_vectors.append(tf.expand_dims(question_hidden, 0))
hidden_vectors = tf.concat(0, hidden_vectors)
return question_hidden, hidden_vectors
def history_recurrent_step(self, curr_hprev, hprev):
#A single RNN step for controller or history RNN
return tf.tanh(
tf.matmul(
tf.concat(1, [hprev, curr_hprev]), self.params[
"history_recurrent"])) + self.params["history_recurrent_bias"]
def question_number_softmax(self, hidden_vectors):
#Attention on quetsion to decide the question number to passed to comparison ops
def compute_ans(op_embedding, comparison):
op_embedding = tf.expand_dims(op_embedding, 0)
#dot product of operation embedding with hidden state to the left of the number occurence
first = tf.transpose(
tf.matmul(op_embedding,
tf.transpose(
tf.reduce_sum(hidden_vectors * tf.tile(
tf.expand_dims(
tf.transpose(self.batch_ordinal_question), 2),
[1, 1, self.utility.FLAGS.embedding_dims]), 0))))
second = self.batch_question_number_one_mask + tf.transpose(
tf.matmul(op_embedding,
tf.transpose(
tf.reduce_sum(hidden_vectors * tf.tile(
tf.expand_dims(
tf.transpose(self.batch_ordinal_question_one), 2
), [1, 1, self.utility.FLAGS.embedding_dims]), 0))))
question_number_softmax = tf.nn.softmax(tf.concat(1, [first, second]))
if (self.mode == "test"):
cond = tf.equal(question_number_softmax,
tf.reshape(
tf.reduce_max(question_number_softmax, 1),
[self.batch_size, 1]))
question_number_softmax = tf.select(
cond,
tf.fill(tf.shape(question_number_softmax), 1.0),
tf.fill(tf.shape(question_number_softmax), 0.0))
question_number_softmax = tf.cast(question_number_softmax,
self.data_type)
ans = tf.reshape(
tf.reduce_sum(question_number_softmax * tf.concat(
1, [self.batch_question_number, self.batch_question_number_one]),
1), [self.batch_size, 1])
return ans
def compute_op_position(op_name):
for i in range(len(self.utility.operations_set)):
if (op_name == self.utility.operations_set[i]):
return i
def compute_question_number(op_name):
op_embedding = tf.nn.embedding_lookup(self.params_unit,
compute_op_position(op_name))
return compute_ans(op_embedding, op_name)
curr_greater_question_number = compute_question_number("greater")
curr_lesser_question_number = compute_question_number("lesser")
curr_geq_question_number = compute_question_number("geq")
curr_leq_question_number = compute_question_number("leq")
return curr_greater_question_number, curr_lesser_question_number, curr_geq_question_number, curr_leq_question_number
def perform_attention(self, context_vector, hidden_vectors, length, mask):
#Performs attention on hiddent_vectors using context vector
context_vector = tf.tile(
tf.expand_dims(context_vector, 0), [length, 1, 1]) #time * bs * d
attention_softmax = tf.nn.softmax(
tf.transpose(tf.reduce_sum(context_vector * hidden_vectors, 2)) +
mask) #batch_size * time
attention_softmax = tf.tile(
tf.expand_dims(tf.transpose(attention_softmax), 2),
[1, 1, self.embedding_dims])
ans_vector = tf.reduce_sum(attention_softmax * hidden_vectors, 0)
return ans_vector
#computes embeddings for column names using parameters of question module
def get_column_hidden_vectors(self):
#vector representations for the column names
self.column_hidden_vectors = tf.reduce_sum(
nn_utils.get_embedding(self.batch_number_column_names, self.utility,
self.params), 2)
self.word_column_hidden_vectors = tf.reduce_sum(
nn_utils.get_embedding(self.batch_word_column_names, self.utility,
self.params), 2)
def create_summary_embeddings(self):
#embeddings for each text entry in the table using parameters of the question module
self.summary_text_entry_embeddings = tf.reduce_sum(
tf.expand_dims(self.batch_exact_match, 3) * tf.expand_dims(
tf.expand_dims(
tf.expand_dims(
nn_utils.get_embedding(self.utility.entry_match_token_id,
self.utility, self.params), 0), 1),
2), 2)
def compute_column_softmax(self, column_controller_vector, time_step):
#compute softmax over all the columns using column controller vector
column_controller_vector = tf.tile(
tf.expand_dims(column_controller_vector, 1),
[1, self.num_cols + self.num_word_cols, 1]) #max_cols * bs * d
column_controller_vector = nn_utils.apply_dropout(
column_controller_vector, self.utility.FLAGS.dropout, self.mode)
self.full_column_hidden_vectors = tf.concat(
1, [self.column_hidden_vectors, self.word_column_hidden_vectors])
self.full_column_hidden_vectors += self.summary_text_entry_embeddings
self.full_column_hidden_vectors = nn_utils.apply_dropout(
self.full_column_hidden_vectors, self.utility.FLAGS.dropout, self.mode)
column_logits = tf.reduce_sum(
column_controller_vector * self.full_column_hidden_vectors, 2) + (
self.params["word_match_feature_column_name"] *
self.batch_column_exact_match) + self.full_column_mask
column_softmax = tf.nn.softmax(column_logits) #batch_size * max_cols
return column_softmax
def compute_first_or_last(self, select, first=True):
#perform first ot last operation on row select with probabilistic row selection
answer = tf.zeros_like(select)
running_sum = tf.zeros([self.batch_size, 1], self.data_type)
for i in range(self.max_elements):
if (first):
current = tf.slice(select, [0, i], [self.batch_size, 1])
else:
current = tf.slice(select, [0, self.max_elements - 1 - i],
[self.batch_size, 1])
curr_prob = current * (1 - running_sum)
curr_prob = curr_prob * tf.cast(curr_prob >= 0.0, self.data_type)
running_sum += curr_prob
temp_ans = []
curr_prob = tf.expand_dims(tf.reshape(curr_prob, [self.batch_size]), 0)
for i_ans in range(self.max_elements):
if (not (first) and i_ans == self.max_elements - 1 - i):
temp_ans.append(curr_prob)
elif (first and i_ans == i):
temp_ans.append(curr_prob)
else:
temp_ans.append(tf.zeros_like(curr_prob))
temp_ans = tf.transpose(tf.concat(0, temp_ans))
answer += temp_ans
return answer
def make_hard_softmax(self, softmax):
#converts soft selection to hard selection. used at test time
cond = tf.equal(
softmax, tf.reshape(tf.reduce_max(softmax, 1), [self.batch_size, 1]))
softmax = tf.select(
cond, tf.fill(tf.shape(softmax), 1.0), tf.fill(tf.shape(softmax), 0.0))
softmax = tf.cast(softmax, self.data_type)
return softmax
def compute_max_or_min(self, select, maxi=True):
#computes the argmax and argmin of a column with probabilistic row selection
answer = tf.zeros([
self.batch_size, self.num_cols + self.num_word_cols, self.max_elements
], self.data_type)
sum_prob = tf.zeros([self.batch_size, self.num_cols + self.num_word_cols],
self.data_type)
for j in range(self.max_elements):
if (maxi):
curr_pos = j
else:
curr_pos = self.max_elements - 1 - j
select_index = tf.slice(self.full_processed_sorted_index_column,
[0, 0, curr_pos], [self.batch_size, -1, 1])
select_mask = tf.equal(
tf.tile(
tf.expand_dims(
tf.tile(
tf.expand_dims(tf.range(self.max_elements), 0),
[self.batch_size, 1]), 1),
[1, self.num_cols + self.num_word_cols, 1]), select_index)
curr_prob = tf.expand_dims(select, 1) * tf.cast(
select_mask, self.data_type) * self.select_bad_number_mask
curr_prob = curr_prob * tf.expand_dims((1 - sum_prob), 2)
curr_prob = curr_prob * tf.expand_dims(
tf.cast((1 - sum_prob) > 0.0, self.data_type), 2)
answer = tf.select(select_mask, curr_prob, answer)
sum_prob += tf.reduce_sum(curr_prob, 2)
return answer
def perform_operations(self, softmax, full_column_softmax, select,
prev_select_1, curr_pass):
#performs all the 15 operations. computes scalar output, lookup answer and row selector
column_softmax = tf.slice(full_column_softmax, [0, 0],
[self.batch_size, self.num_cols])
word_column_softmax = tf.slice(full_column_softmax, [0, self.num_cols],
[self.batch_size, self.num_word_cols])
init_max = self.compute_max_or_min(select, maxi=True)
init_min = self.compute_max_or_min(select, maxi=False)
#operations that are column independent
count = tf.reshape(tf.reduce_sum(select, 1), [self.batch_size, 1])
select_full_column_softmax = tf.tile(
tf.expand_dims(full_column_softmax, 2),
[1, 1, self.max_elements
]) #BS * (max_cols + max_word_cols) * max_elements
select_word_column_softmax = tf.tile(
tf.expand_dims(word_column_softmax, 2),
[1, 1, self.max_elements]) #BS * max_word_cols * max_elements
select_greater = tf.reduce_sum(
self.init_select_greater * select_full_column_softmax,
1) * self.batch_question_number_mask #BS * max_elements
select_lesser = tf.reduce_sum(
self.init_select_lesser * select_full_column_softmax,
1) * self.batch_question_number_mask #BS * max_elements
select_geq = tf.reduce_sum(
self.init_select_geq * select_full_column_softmax,
1) * self.batch_question_number_mask #BS * max_elements
select_leq = tf.reduce_sum(
self.init_select_leq * select_full_column_softmax,
1) * self.batch_question_number_mask #BS * max_elements
select_max = tf.reduce_sum(init_max * select_full_column_softmax,
1) #BS * max_elements
select_min = tf.reduce_sum(init_min * select_full_column_softmax,
1) #BS * max_elements
select_prev = tf.concat(1, [
tf.slice(select, [0, 1], [self.batch_size, self.max_elements - 1]),
tf.cast(tf.zeros([self.batch_size, 1]), self.data_type)
])
select_next = tf.concat(1, [
tf.cast(tf.zeros([self.batch_size, 1]), self.data_type), tf.slice(
select, [0, 0], [self.batch_size, self.max_elements - 1])
])
select_last_rs = self.compute_first_or_last(select, False)
select_first_rs = self.compute_first_or_last(select, True)
select_word_match = tf.reduce_sum(self.batch_exact_match *
select_full_column_softmax, 1)
select_group_by_max = tf.reduce_sum(self.batch_group_by_max *
select_full_column_softmax, 1)
length_content = 1
length_select = 13
length_print = 1
values = tf.concat(1, [count])
softmax_content = tf.slice(softmax, [0, 0],
[self.batch_size, length_content])
#compute scalar output
output = tf.reduce_sum(tf.mul(softmax_content, values), 1)
#compute lookup answer
softmax_print = tf.slice(softmax, [0, length_content + length_select],
[self.batch_size, length_print])
curr_print = select_full_column_softmax * tf.tile(
tf.expand_dims(select, 1),
[1, self.num_cols + self.num_word_cols, 1
]) #BS * max_cols * max_elements (conisders only column)
self.batch_lookup_answer = curr_print * tf.tile(
tf.expand_dims(softmax_print, 2),
[1, self.num_cols + self.num_word_cols, self.max_elements
]) #BS * max_cols * max_elements
self.batch_lookup_answer = self.batch_lookup_answer * self.select_full_mask
#compute row select
softmax_select = tf.slice(softmax, [0, length_content],
[self.batch_size, length_select])
select_lists = [
tf.expand_dims(select_prev, 1), tf.expand_dims(select_next, 1),
tf.expand_dims(select_first_rs, 1), tf.expand_dims(select_last_rs, 1),
tf.expand_dims(select_group_by_max, 1),
tf.expand_dims(select_greater, 1), tf.expand_dims(select_lesser, 1),
tf.expand_dims(select_geq, 1), tf.expand_dims(select_leq, 1),
tf.expand_dims(select_max, 1), tf.expand_dims(select_min, 1),
tf.expand_dims(select_word_match, 1),
tf.expand_dims(self.reset_select, 1)
]
select = tf.reduce_sum(
tf.tile(tf.expand_dims(softmax_select, 2), [1, 1, self.max_elements]) *
tf.concat(1, select_lists), 1)
select = select * self.select_whole_mask
return output, select
def one_pass(self, select, question_embedding, hidden_vectors, hprev,
prev_select_1, curr_pass):
#Performs one timestep which involves selecting an operation and a column
attention_vector = self.perform_attention(
hprev, hidden_vectors, self.question_length,
self.batch_question_attention_mask) #batch_size * embedding_dims
controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["controller_prev"]) + tf.matmul(
tf.concat(1, [question_embedding, attention_vector]), self.params[
"controller"]))
column_controller_vector = tf.nn.relu(
tf.matmul(hprev, self.params["column_controller_prev"]) + tf.matmul(
tf.concat(1, [question_embedding, attention_vector]), self.params[
"column_controller"]))
controller_vector = nn_utils.apply_dropout(
controller_vector, self.utility.FLAGS.dropout, self.mode)
self.operation_logits = tf.matmul(controller_vector,
tf.transpose(self.params_unit))
softmax = tf.nn.softmax(self.operation_logits)
soft_softmax = softmax
#compute column softmax: bs * max_columns
weighted_op_representation = tf.transpose(
tf.matmul(tf.transpose(self.params_unit), tf.transpose(softmax)))
column_controller_vector = tf.nn.relu(
tf.matmul(
tf.concat(1, [
column_controller_vector, weighted_op_representation
]), self.params["break_conditional"]))
full_column_softmax = self.compute_column_softmax(column_controller_vector,
curr_pass)
soft_column_softmax = full_column_softmax
if (self.mode == "test"):
full_column_softmax = self.make_hard_softmax(full_column_softmax)
softmax = self.make_hard_softmax(softmax)
output, select = self.perform_operations(softmax, full_column_softmax,
select, prev_select_1, curr_pass)
return output, select, softmax, soft_softmax, full_column_softmax, soft_column_softmax
def compute_lookup_error(self, val):
#computes lookup error.
cond = tf.equal(self.batch_print_answer, val)
inter = tf.select(
cond, self.init_print_error,
tf.tile(
tf.reshape(tf.constant(1e10, self.data_type), [1, 1, 1]), [
self.batch_size, self.utility.FLAGS.max_word_cols +
self.utility.FLAGS.max_number_cols,
self.utility.FLAGS.max_elements
]))
return tf.reduce_min(tf.reduce_min(inter, 1), 1) * tf.cast(
tf.greater(
tf.reduce_sum(tf.reduce_sum(tf.cast(cond, self.data_type), 1), 1),
0.0), self.data_type)
def soft_min(self, x, y):
return tf.maximum(-1.0 * (1 / (
self.utility.FLAGS.soft_min_value + 0.0)) * tf.log(
tf.exp(-self.utility.FLAGS.soft_min_value * x) + tf.exp(
-self.utility.FLAGS.soft_min_value * y)), tf.zeros_like(x))
def error_computation(self):
#computes the error of each example in a batch
math_error = 0.5 * tf.square(tf.sub(self.scalar_output, self.batch_answer))
#scale math error
math_error = math_error / self.rows
math_error = tf.minimum(math_error, self.utility.FLAGS.max_math_error *
tf.ones(tf.shape(math_error), self.data_type))
self.init_print_error = tf.select(
self.batch_gold_select, -1 * tf.log(self.batch_lookup_answer + 1e-300 +
self.invert_select_full_mask), -1 *
tf.log(1 - self.batch_lookup_answer)) * self.select_full_mask
print_error_1 = self.init_print_error * tf.cast(
tf.equal(self.batch_print_answer, 0.0), self.data_type)
print_error = tf.reduce_sum(tf.reduce_sum((print_error_1), 1), 1)
for val in range(1, 58):
print_error += self.compute_lookup_error(val + 0.0)
print_error = print_error * self.utility.FLAGS.print_cost / self.num_entries
if (self.mode == "train"):
error = tf.select(
tf.logical_and(
tf.not_equal(self.batch_answer, 0.0),
tf.not_equal(
tf.reduce_sum(tf.reduce_sum(self.batch_print_answer, 1), 1),
0.0)),
self.soft_min(math_error, print_error),
tf.select(
tf.not_equal(self.batch_answer, 0.0), math_error, print_error))
else:
error = tf.select(
tf.logical_and(
tf.equal(self.scalar_output, 0.0),
tf.equal(
tf.reduce_sum(tf.reduce_sum(self.batch_lookup_answer, 1), 1),
0.0)),
tf.ones_like(math_error),
tf.select(
tf.equal(self.scalar_output, 0.0), print_error, math_error))
return error
def batch_process(self):
#Computes loss and fraction of correct examples in a batch.
self.params_unit = nn_utils.apply_dropout(
self.params["unit"], self.utility.FLAGS.dropout, self.mode)
batch_size = self.batch_size
max_passes = self.max_passes
num_timesteps = 1
max_elements = self.max_elements
select = tf.cast(
tf.fill([self.batch_size, max_elements], 1.0), self.data_type)
hprev = tf.cast(
tf.fill([self.batch_size, self.embedding_dims], 0.0),
self.data_type) #running sum of the hidden states of the model
output = tf.cast(tf.fill([self.batch_size, 1], 0.0),
self.data_type) #output of the model
correct = tf.cast(
tf.fill([1], 0.0), self.data_type
) #to compute accuracy, returns number of correct examples for this batch
total_error = 0.0
prev_select_1 = tf.zeros_like(select)
self.create_summary_embeddings()
self.get_column_hidden_vectors()
#get question embedding
question_embedding, hidden_vectors = self.LSTM_question_embedding(
self.batch_question, self.question_length)
#compute arguments for comparison operation
greater_question_number, lesser_question_number, geq_question_number, leq_question_number = self.question_number_softmax(
hidden_vectors)
self.init_select_greater = tf.cast(
tf.greater(self.full_processed_column,
tf.expand_dims(greater_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_lesser = tf.cast(
tf.less(self.full_processed_column,
tf.expand_dims(lesser_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_geq = tf.cast(
tf.greater_equal(self.full_processed_column,
tf.expand_dims(geq_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_leq = tf.cast(
tf.less_equal(self.full_processed_column,
tf.expand_dims(leq_question_number, 2)), self.
data_type) * self.select_bad_number_mask #bs * max_cols * max_elements
self.init_select_word_match = 0
if (self.utility.FLAGS.rnn_dropout > 0.0):
if (self.mode == "train"):
history_rnn_dropout_mask = tf.cast(
tf.random_uniform(
tf.shape(hprev), minval=0.0, maxval=1.0) <
self.utility.FLAGS.rnn_dropout,
self.data_type) / self.utility.FLAGS.rnn_dropout
else:
history_rnn_dropout_mask = tf.ones_like(hprev)
select = select * self.select_whole_mask
self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type)
#Perform max_passes and at each pass select operation and column
for curr_pass in range(max_passes):
print "step: ", curr_pass
output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass(
select, question_embedding, hidden_vectors, hprev, prev_select_1,
curr_pass)
prev_select_1 = select
#compute input to history RNN
input_op = tf.transpose(
tf.matmul(
tf.transpose(self.params_unit), tf.transpose(
soft_softmax))) #weighted average of emebdding of operations
input_col = tf.reduce_sum(
tf.expand_dims(soft_column_softmax, 2) *
self.full_column_hidden_vectors, 1)
history_input = tf.concat(1, [input_op, input_col])
history_input = nn_utils.apply_dropout(
history_input, self.utility.FLAGS.dropout, self.mode)
hprev = self.history_recurrent_step(history_input, hprev)
if (self.utility.FLAGS.rnn_dropout > 0.0):
hprev = hprev * history_rnn_dropout_mask
self.scalar_output = output
error = self.error_computation()
cond = tf.less(error, 0.0001, name="cond")
correct_add = tf.select(
cond, tf.fill(tf.shape(cond), 1.0), tf.fill(tf.shape(cond), 0.0))
correct = tf.reduce_sum(correct_add)
error = error / batch_size
total_error = tf.reduce_sum(error)
total_correct = correct / batch_size
return total_error, total_correct
def compute_error(self):
#Sets mask variables and performs batch processing
self.batch_gold_select = self.batch_print_answer > 0.0
self.full_column_mask = tf.concat(
1, [self.batch_number_column_mask, self.batch_word_column_mask])
self.full_processed_column = tf.concat(
1,
[self.batch_processed_number_column, self.batch_processed_word_column])
self.full_processed_sorted_index_column = tf.concat(1, [
self.batch_processed_sorted_index_number_column,
self.batch_processed_sorted_index_word_column
])
self.select_bad_number_mask = tf.cast(
tf.logical_and(
tf.not_equal(self.full_processed_column,
self.utility.FLAGS.pad_int),
tf.not_equal(self.full_processed_column,
self.utility.FLAGS.bad_number_pre_process)),
self.data_type)
self.select_mask = tf.cast(
tf.logical_not(
tf.equal(self.batch_number_column, self.utility.FLAGS.pad_int)),
self.data_type)
self.select_word_mask = tf.cast(
tf.logical_not(
tf.equal(self.batch_word_column_entry_mask,
self.utility.dummy_token_id)), self.data_type)
self.select_full_mask = tf.concat(
1, [self.select_mask, self.select_word_mask])
self.select_whole_mask = tf.maximum(
tf.reshape(
tf.slice(self.select_mask, [0, 0, 0],
[self.batch_size, 1, self.max_elements]),
[self.batch_size, self.max_elements]),
tf.reshape(
tf.slice(self.select_word_mask, [0, 0, 0],
[self.batch_size, 1, self.max_elements]),
[self.batch_size, self.max_elements]))
self.invert_select_full_mask = tf.cast(
tf.concat(1, [
tf.equal(self.batch_number_column, self.utility.FLAGS.pad_int),
tf.equal(self.batch_word_column_entry_mask,
self.utility.dummy_token_id)
]), self.data_type)
self.batch_lookup_answer = tf.zeros(tf.shape(self.batch_gold_select))
self.reset_select = self.select_whole_mask
self.rows = tf.reduce_sum(self.select_whole_mask, 1)
self.num_entries = tf.reshape(
tf.reduce_sum(tf.reduce_sum(self.select_full_mask, 1), 1),
[self.batch_size])
self.final_error, self.final_correct = self.batch_process()
return self.final_error
def create_graph(self, params, global_step):
#Creates the graph to compute error, gradient computation and updates parameters
self.params = params
batch_size = self.batch_size
learning_rate = tf.cast(self.utility.FLAGS.learning_rate, self.data_type)
self.total_cost = self.compute_error()
optimize_params = self.params.values()
optimize_names = self.params.keys()
print "optimize params ", optimize_names
if (self.utility.FLAGS.l2_regularizer > 0.0):
reg_cost = 0.0
for ind_param in self.params.keys():
reg_cost += tf.nn.l2_loss(self.params[ind_param])
self.total_cost += self.utility.FLAGS.l2_regularizer * reg_cost
grads = tf.gradients(self.total_cost, optimize_params, name="gradients")
grad_norm = 0.0
for p, name in zip(grads, optimize_names):
print "grads: ", p, name
if isinstance(p, tf.IndexedSlices):
grad_norm += tf.reduce_sum(p.values * p.values)
elif not (p == None):
grad_norm += tf.reduce_sum(p * p)
grad_norm = tf.sqrt(grad_norm)
max_grad_norm = np.float32(self.utility.FLAGS.clip_gradients).astype(
self.utility.np_data_type[self.utility.FLAGS.data_type])
grad_scale = tf.minimum(
tf.cast(1.0, self.data_type), max_grad_norm / grad_norm)
clipped_grads = list()
for p in grads:
if isinstance(p, tf.IndexedSlices):
tmp = p.values * grad_scale
clipped_grads.append(tf.IndexedSlices(tmp, p.indices))
elif not (p == None):
clipped_grads.append(p * grad_scale)
else:
clipped_grads.append(p)
grads = clipped_grads
self.global_step = global_step
params_list = self.params.values()
params_list.append(self.global_step)
adam = tf.train.AdamOptimizer(
learning_rate,
epsilon=tf.cast(self.utility.FLAGS.eps, self.data_type),
use_locking=True)
self.step = adam.apply_gradients(zip(grads, optimize_params),
global_step=self.global_step)
self.init_op = tf.initialize_all_variables()
neural_programmer.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Implementation of the Neural Programmer model described in https://openreview.net/pdf?id=ry2YOrcge
This file calls functions to load & pre-process data, construct the TF graph
and performs training or evaluation as specified by the flag evaluator_job
Author: aneelakantan (Arvind Neelakantan)
"""
import time
from random import Random
import numpy as np
import tensorflow as tf
import model
import wiki_data
import parameters
import data_utils
tf.flags.DEFINE_integer("train_steps", 100001, "Number of steps to train")
tf.flags.DEFINE_integer("eval_cycle", 500,
"Evaluate model at every eval_cycle steps")
tf.flags.DEFINE_integer("max_elements", 100,
"maximum rows that are considered for processing")
tf.flags.DEFINE_integer(
"max_number_cols", 15,
"maximum number columns that are considered for processing")
tf.flags.DEFINE_integer(
"max_word_cols", 25,
"maximum number columns that are considered for processing")
tf.flags.DEFINE_integer("question_length", 62, "maximum question length")
tf.flags.DEFINE_integer("max_entry_length", 1, "")
tf.flags.DEFINE_integer("max_passes", 4, "number of operation passes")
tf.flags.DEFINE_integer("embedding_dims", 256, "")
tf.flags.DEFINE_integer("batch_size", 20, "")
tf.flags.DEFINE_float("clip_gradients", 1.0, "")
tf.flags.DEFINE_float("eps", 1e-6, "")
tf.flags.DEFINE_float("param_init", 0.1, "")
tf.flags.DEFINE_float("learning_rate", 0.001, "")
tf.flags.DEFINE_float("l2_regularizer", 0.0001, "")
tf.flags.DEFINE_float("print_cost", 50.0,
"weighting factor in the objective function")
tf.flags.DEFINE_string("job_id", "temp", """job id""")
tf.flags.DEFINE_string("output_dir", "../model/",
"""output_dir""")
tf.flags.DEFINE_string("data_dir", "../data/",
"""data_dir""")
tf.flags.DEFINE_integer("write_every", 500, "wrtie every N")
tf.flags.DEFINE_integer("param_seed", 150, "")
tf.flags.DEFINE_integer("python_seed", 200, "")
tf.flags.DEFINE_float("dropout", 0.8, "dropout keep probability")
tf.flags.DEFINE_float("rnn_dropout", 0.9,
"dropout keep probability for rnn connections")
tf.flags.DEFINE_float("pad_int", -20000.0,
"number columns are padded with pad_int")
tf.flags.DEFINE_string("data_type", "double", "float or double")
tf.flags.DEFINE_float("word_dropout_prob", 0.9, "word dropout keep prob")
tf.flags.DEFINE_integer("word_cutoff", 10, "")
tf.flags.DEFINE_integer("vocab_size", 10800, "")
tf.flags.DEFINE_boolean("evaluator_job", False,
"wehther to run as trainer/evaluator")
tf.flags.DEFINE_float(
"bad_number_pre_process", -200000.0,
"number that is added to a corrupted table entry in a number column")
tf.flags.DEFINE_float("max_math_error", 3.0,
"max square loss error that is considered")
tf.flags.DEFINE_float("soft_min_value", 5.0, "")
FLAGS = tf.flags.FLAGS
class Utility:
#holds FLAGS and other variables that are used in different files
def __init__(self):
global FLAGS
self.FLAGS = FLAGS
self.unk_token = "UNK"
self.entry_match_token = "entry_match"
self.column_match_token = "column_match"
self.dummy_token = "dummy_token"
self.tf_data_type = {}
self.tf_data_type["double"] = tf.float64
self.tf_data_type["float"] = tf.float32
self.np_data_type = {}
self.np_data_type["double"] = np.float64
self.np_data_type["float"] = np.float32
self.operations_set = ["count"] + [
"prev", "next", "first_rs", "last_rs", "group_by_max", "greater",
"lesser", "geq", "leq", "max", "min", "word-match"
] + ["reset_select"] + ["print"]
self.word_ids = {}
self.reverse_word_ids = {}
self.word_count = {}
self.random = Random(FLAGS.python_seed)
def evaluate(sess, data, batch_size, graph, i):
#computes accuracy
num_examples = 0.0
gc = 0.0
for j in range(0, len(data) - batch_size + 1, batch_size):
[ct] = sess.run([graph.final_correct],
feed_dict=data_utils.generate_feed_dict(data, j, batch_size,
graph))
gc += ct * batch_size
num_examples += batch_size
print "dev set accuracy after ", i, " : ", gc / num_examples
print num_examples, len(data)
print "--------"
def Train(graph, utility, batch_size, train_data, sess, model_dir,
saver):
#performs training
curr = 0
train_set_loss = 0.0
utility.random.shuffle(train_data)
start = time.time()
for i in range(utility.FLAGS.train_steps):
curr_step = i
if (i > 0 and i % FLAGS.write_every == 0):
model_file = model_dir + "/model_" + str(i)
saver.save(sess, model_file)
if curr + batch_size >= len(train_data):
curr = 0
utility.random.shuffle(train_data)
step, cost_value = sess.run(
[graph.step, graph.total_cost],
feed_dict=data_utils.generate_feed_dict(
train_data, curr, batch_size, graph, train=True, utility=utility))
curr = curr + batch_size
train_set_loss += cost_value
if (i > 0 and i % FLAGS.eval_cycle == 0):
end = time.time()
time_taken = end - start
print "step ", i, " ", time_taken, " seconds "
start = end
print " printing train set loss: ", train_set_loss / utility.FLAGS.eval_cycle
train_set_loss = 0.0
def master(train_data, dev_data, utility):
#creates TF graph and calls trainer or evaluator
batch_size = utility.FLAGS.batch_size
model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/"
#create all paramters of the model
param_class = parameters.Parameters(utility)
params, global_step, init = param_class.parameters(utility)
key = "test" if (FLAGS.evaluator_job) else "train"
graph = model.Graph(utility, batch_size, utility.FLAGS.max_passes, mode=key)
graph.create_graph(params, global_step)
prev_dev_error = 0.0
final_loss = 0.0
final_accuracy = 0.0
#start session
with tf.Session() as sess:
sess.run(init.name)
sess.run(graph.init_op.name)
to_save = params.copy()
saver = tf.train.Saver(to_save, max_to_keep=500)
if (FLAGS.evaluator_job):
while True:
selected_models = {}
file_list = tf.gfile.ListDirectory(model_dir)
for model_file in file_list:
if ("checkpoint" in model_file or "index" in model_file or
"meta" in model_file):
continue
if ("data" in model_file):
model_file = model_file.split(".")[0]
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
selected_models[model_step] = model_file
file_list = sorted(selected_models.items(), key=lambda x: x[0])
if (len(file_list) > 0):
file_list = file_list[0:len(file_list) - 1]
print "list of models: ", file_list
for model_file in file_list:
model_file = model_file[1]
print "restoring: ", model_file
saver.restore(sess, model_dir + "/" + model_file)
model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1])
print "evaluating on dev ", model_file, model_step
evaluate(sess, dev_data, batch_size, graph, model_step)
else:
ckpt = tf.train.get_checkpoint_state(model_dir)
print "model dir: ", model_dir
if (not (tf.gfile.IsDirectory(model_dir))):
print "create dir: ", model_dir
tf.gfile.MkDir(model_dir)
Train(graph, utility, batch_size, train_data, sess, model_dir,
saver)
def main(args):
utility = Utility()
train_name = "random-split-1-train.examples"
dev_name = "random-split-1-dev.examples"
test_name = "pristine-unseen-tables.examples"
#load data
dat = wiki_data.WikiQuestionGenerator(train_name, dev_name, test_name, FLAGS.data_dir)
train_data, dev_data, test_data = dat.load()
utility.words = []
utility.word_ids = {}
utility.reverse_word_ids = {}
#construct vocabulary
data_utils.construct_vocab(train_data, utility)
data_utils.construct_vocab(dev_data, utility, True)
data_utils.construct_vocab(test_data, utility, True)
data_utils.add_special_words(utility)
data_utils.perform_word_cutoff(utility)
#convert data to int format and pad the inputs
train_data = data_utils.complete_wiki_processing(train_data, utility, True)
dev_data = data_utils.complete_wiki_processing(dev_data, utility, False)
test_data = data_utils.complete_wiki_processing(test_data, utility, False)
print "# train examples ", len(train_data)
print "# dev examples ", len(dev_data)
print "# test examples ", len(test_data)
print "running open source"
#construct TF graph and train or evaluate
master(train_data, dev_data, utility)
if __name__ == "__main__":
tf.app.run()
nn_utils.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Author: aneelakantan (Arvind Neelakantan)
"""
import tensorflow as tf
def get_embedding(word, utility, params):
return tf.nn.embedding_lookup(params["word"], word)
def apply_dropout(x, dropout_rate, mode):
if (dropout_rate > 0.0):
if (mode == "train"):
x = tf.nn.dropout(x, dropout_rate)
else:
x = x
return x
def LSTMCell(x, mprev, cprev, key, params):
"""Create an LSTM cell.
Implements the equations in pg.2 from
"Long Short-Term Memory Based Recurrent Neural Network Architectures
For Large Vocabulary Speech Recognition",
Hasim Sak, Andrew Senior, Francoise Beaufays.
Args:
w: A dictionary of the weights and optional biases as returned
by LSTMParametersSplit().
x: Inputs to this cell.
mprev: m_{t-1}, the recurrent activations (same as the output)
from the previous cell.
cprev: c_{t-1}, the cell activations from the previous cell.
keep_prob: Keep probability on the input and the outputs of a cell.
Returns:
m: Outputs of this cell.
c: Cell Activations.
"""
i = tf.matmul(x, params[key + "_ix"]) + tf.matmul(mprev, params[key + "_im"])
i = tf.nn.bias_add(i, params[key + "_i"])
f = tf.matmul(x, params[key + "_fx"]) + tf.matmul(mprev, params[key + "_fm"])
f = tf.nn.bias_add(f, params[key + "_f"])
c = tf.matmul(x, params[key + "_cx"]) + tf.matmul(mprev, params[key + "_cm"])
c = tf.nn.bias_add(c, params[key + "_c"])
o = tf.matmul(x, params[key + "_ox"]) + tf.matmul(mprev, params[key + "_om"])
o = tf.nn.bias_add(o, params[key + "_o"])
i = tf.sigmoid(i, name="i_gate")
f = tf.sigmoid(f, name="f_gate")
o = tf.sigmoid(o, name="o_gate")
c = f * cprev + i * tf.tanh(c)
m = o * c
return m, c
parameters.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Author: aneelakantan (Arvind Neelakantan)
"""
import numpy as np
import tensorflow as tf
class Parameters:
def __init__(self, u):
self.utility = u
self.init_seed_counter = 0
self.word_init = {}
def parameters(self, utility):
params = {}
inits = []
embedding_dims = self.utility.FLAGS.embedding_dims
params["unit"] = tf.Variable(
self.RandomUniformInit([len(utility.operations_set), embedding_dims]))
params["word"] = tf.Variable(
self.RandomUniformInit([utility.FLAGS.vocab_size, embedding_dims]))
params["word_match_feature_column_name"] = tf.Variable(
self.RandomUniformInit([1]))
params["controller"] = tf.Variable(
self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
params["column_controller"] = tf.Variable(
self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
params["column_controller_prev"] = tf.Variable(
self.RandomUniformInit([embedding_dims, embedding_dims]))
params["controller_prev"] = tf.Variable(
self.RandomUniformInit([embedding_dims, embedding_dims]))
global_step = tf.Variable(1, name="global_step")
#weigths of question and history RNN (or LSTM)
key_list = ["question_lstm"]
for key in key_list:
# Weights going from inputs to nodes.
for wgts in ["ix", "fx", "cx", "ox"]:
params[key + "_" + wgts] = tf.Variable(
self.RandomUniformInit([embedding_dims, embedding_dims]))
# Weights going from nodes to nodes.
for wgts in ["im", "fm", "cm", "om"]:
params[key + "_" + wgts] = tf.Variable(
self.RandomUniformInit([embedding_dims, embedding_dims]))
#Biases for the gates and cell
for bias in ["i", "f", "c", "o"]:
if (bias == "f"):
print "forget gate bias"
params[key + "_" + bias] = tf.Variable(
tf.random_uniform([embedding_dims], 1.0, 1.1, self.utility.
tf_data_type[self.utility.FLAGS.data_type]))
else:
params[key + "_" + bias] = tf.Variable(
self.RandomUniformInit([embedding_dims]))
params["history_recurrent"] = tf.Variable(
self.RandomUniformInit([3 * embedding_dims, embedding_dims]))
params["history_recurrent_bias"] = tf.Variable(
self.RandomUniformInit([1, embedding_dims]))
params["break_conditional"] = tf.Variable(
self.RandomUniformInit([2 * embedding_dims, embedding_dims]))
init = tf.initialize_all_variables()
return params, global_step, init
def RandomUniformInit(self, shape):
"""Returns a RandomUniform Tensor between -param_init and param_init."""
param_seed = self.utility.FLAGS.param_seed
self.init_seed_counter += 1
return tf.random_uniform(
shape, -1.0 *
(np.float32(self.utility.FLAGS.param_init)
).astype(self.utility.np_data_type[self.utility.FLAGS.data_type]),
(np.float32(self.utility.FLAGS.param_init)
).astype(self.utility.np_data_type[self.utility.FLAGS.data_type]),
self.utility.tf_data_type[self.utility.FLAGS.data_type],
param_seed + self.init_seed_counter)
wiki_data.py 0 → 100755
View file @ 124a501d
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Loads the WikiQuestions dataset.
An example consists of question, table. Additionally, we store the processed
columns which store the entries after performing number, date and other
preprocessing as done in the baseline.
columns, column names and processed columns are split into word and number
columns.
lookup answer (or matrix) is also split into number and word lookup matrix
Author: aneelakantan (Arvind Neelakantan)
"""
import math
import os
import re
import numpy as np
import unicodedata as ud
import tensorflow as tf
bad_number = -200000.0 #number that is added to a corrupted table entry in a number column
def is_nan_or_inf(number):
return math.isnan(number) or math.isinf(number)
def strip_accents(s):
u = unicode(s, "utf-8")
u_new = ''.join(c for c in ud.normalize('NFKD', u) if ud.category(c) != 'Mn')
return u_new.encode("utf-8")
def correct_unicode(string):
string = strip_accents(string)
string = re.sub("\xc2\xa0", " ", string).strip()
string = re.sub("\xe2\x80\x93", "-", string).strip()
#string = re.sub(ur'[\u0300-\u036F]', "", string)
string = re.sub("‚", ",", string)
string = re.sub("…", "...", string)
#string = re.sub("[·・]", ".", string)
string = re.sub("ˆ", "^", string)
string = re.sub("˜", "~", string)
string = re.sub("‹", "<", string)
string = re.sub("›", ">", string)
#string = re.sub("[‘’´`]", "'", string)
#string = re.sub("[“”«»]", "\"", string)
#string = re.sub("[•†‡]", "", string)
#string = re.sub("[‐‑–—]", "-", string)
string = re.sub(ur'[\u2E00-\uFFFF]', "", string)
string = re.sub("\\s+", " ", string).strip()
return string
def simple_normalize(string):
string = correct_unicode(string)
# Citations
string = re.sub("\[(nb ?)?\d+\]", "", string)
string = re.sub("\*+$", "", string)
# Year in parenthesis
string = re.sub("\(\d* ?-? ?\d*\)", "", string)
string = re.sub("^\"(.*)\"$", "", string)
return string
def full_normalize(string):
#print "an: ", string
string = simple_normalize(string)
# Remove trailing info in brackets
string = re.sub("\[[^\]]*\]", "", string)
# Remove most unicode characters in other languages
string = re.sub(ur'[\u007F-\uFFFF]', "", string.strip())
# Remove trailing info in parenthesis
string = re.sub("\([^)]*\)$", "", string.strip())
string = final_normalize(string)
# Get rid of question marks
string = re.sub("\?", "", string).strip()
# Get rid of trailing colons (usually occur in column titles)
string = re.sub("\:$", " ", string).strip()
# Get rid of slashes
string = re.sub(r"/", " ", string).strip()
string = re.sub(r"\\", " ", string).strip()
# Replace colon, slash, and dash with space
# Note: need better replacement for this when parsing time
string = re.sub(r"\:", " ", string).strip()
string = re.sub("/", " ", string).strip()
string = re.sub("-", " ", string).strip()
# Convert empty strings to UNK
# Important to do this last or near last
if not string:
string = "UNK"
return string
def final_normalize(string):
# Remove leading and trailing whitespace
string = re.sub("\\s+", " ", string).strip()
# Convert entirely to lowercase
string = string.lower()
# Get rid of strangely escaped newline characters
string = re.sub("\\\\n", " ", string).strip()
# Get rid of quotation marks
string = re.sub(r"\"", "", string).strip()
string = re.sub(r"\'", "", string).strip()
string = re.sub(r"`", "", string).strip()
# Get rid of *
string = re.sub("\*", "", string).strip()
return string
def is_number(x):
try:
f = float(x)
return not is_nan_or_inf(f)
except ValueError:
return False
except TypeError:
return False
class WikiExample(object):
def __init__(self, id, question, answer, table_key):
self.question_id = id
self.question = question
self.answer = answer
self.table_key = table_key
self.lookup_matrix = []
self.is_bad_example = False
self.is_word_lookup = False
self.is_ambiguous_word_lookup = False
self.is_number_lookup = False
self.is_number_calc = False
self.is_unknown_answer = False
class TableInfo(object):
def __init__(self, word_columns, word_column_names, word_column_indices,
number_columns, number_column_names, number_column_indices,
processed_word_columns, processed_number_columns, orig_columns):
self.word_columns = word_columns
self.word_column_names = word_column_names
self.word_column_indices = word_column_indices
self.number_columns = number_columns
self.number_column_names = number_column_names
self.number_column_indices = number_column_indices
self.processed_word_columns = processed_word_columns
self.processed_number_columns = processed_number_columns
self.orig_columns = orig_columns
class WikiQuestionLoader(object):
def __init__(self, data_name, root_folder):
self.root_folder = root_folder
self.data_folder = os.path.join(self.root_folder, "data")
self.examples = []
self.data_name = data_name
def num_questions(self):
return len(self.examples)
def load_qa(self):
data_source = os.path.join(self.data_folder, self.data_name)
f = tf.gfile.GFile(data_source, "r")
id_regex = re.compile("\(id ([^\)]*)\)")
for line in f:
id_match = id_regex.search(line)
id = id_match.group(1)
self.examples.append(id)
def load(self):
self.load_qa()
def is_date(word):
if (not (bool(re.search("[a-z0-9]", word, re.IGNORECASE)))):
return False
if (len(word) != 10):
return False
if (word[4] != "-"):
return False
if (word[7] != "-"):
return False
for i in range(len(word)):
if (not (word[i] == "X" or word[i] == "x" or word[i] == "-" or re.search(
"[0-9]", word[i]))):
return False
return True
class WikiQuestionGenerator(object):
def __init__(self, train_name, dev_name, test_name, root_folder):
self.train_name = train_name
self.dev_name = dev_name
self.test_name = test_name
self.train_loader = WikiQuestionLoader(train_name, root_folder)
self.dev_loader = WikiQuestionLoader(dev_name, root_folder)
self.test_loader = WikiQuestionLoader(test_name, root_folder)
self.bad_examples = 0
self.root_folder = root_folder
self.data_folder = os.path.join(self.root_folder, "annotated/data")
self.annotated_examples = {}
self.annotated_tables = {}
self.annotated_word_reject = {}
self.annotated_word_reject["-lrb-"] = 1
self.annotated_word_reject["-rrb-"] = 1
self.annotated_word_reject["UNK"] = 1
def is_money(self, word):
if (not (bool(re.search("[a-z0-9]", word, re.IGNORECASE)))):
return False
for i in range(len(word)):
if (not (word[i] == "E" or word[i] == "." or re.search("[0-9]",
word[i]))):
return False
return True
def remove_consecutive(self, ner_tags, ner_values):
for i in range(len(ner_tags)):
if ((ner_tags[i] == "NUMBER" or ner_tags[i] == "MONEY" or
ner_tags[i] == "PERCENT" or ner_tags[i] == "DATE") and
i + 1 < len(ner_tags) and ner_tags[i] == ner_tags[i + 1] and
ner_values[i] == ner_values[i + 1] and ner_values[i] != ""):
word = ner_values[i]
word = word.replace(">", "").replace("<", "").replace("=", "").replace(
"%", "").replace("~", "").replace("$", "").replace("£", "").replace(
"€", "")
if (re.search("[A-Z]", word) and not (is_date(word)) and not (
self.is_money(word))):
ner_values[i] = "A"
else:
ner_values[i] = ","
return ner_tags, ner_values
def pre_process_sentence(self, tokens, ner_tags, ner_values):
sentence = []
tokens = tokens.split("|")
ner_tags = ner_tags.split("|")
ner_values = ner_values.split("|")
ner_tags, ner_values = self.remove_consecutive(ner_tags, ner_values)
#print "old: ", tokens
for i in range(len(tokens)):
word = tokens[i]
if (ner_values[i] != "" and
(ner_tags[i] == "NUMBER" or ner_tags[i] == "MONEY" or
ner_tags[i] == "PERCENT" or ner_tags[i] == "DATE")):
word = ner_values[i]
word = word.replace(">", "").replace("<", "").replace("=", "").replace(
"%", "").replace("~", "").replace("$", "").replace("£", "").replace(
"€", "")
if (re.search("[A-Z]", word) and not (is_date(word)) and not (
self.is_money(word))):
word = tokens[i]
if (is_number(ner_values[i])):
word = float(ner_values[i])
elif (is_number(word)):
word = float(word)
if (tokens[i] == "score"):
word = "score"
if (is_number(word)):
word = float(word)
if (not (self.annotated_word_reject.has_key(word))):
if (is_number(word) or is_date(word) or self.is_money(word)):
sentence.append(word)
else:
word = full_normalize(word)
if (not (self.annotated_word_reject.has_key(word)) and
bool(re.search("[a-z0-9]", word, re.IGNORECASE))):
m = re.search(",", word)
sentence.append(word.replace(",", ""))
if (len(sentence) == 0):
sentence.append("UNK")
return sentence
def load_annotated_data(self, in_file):
self.annotated_examples = {}
self.annotated_tables = {}
f = tf.gfile.GFile(in_file, "r")
counter = 0
for line in f:
if (counter > 0):
line = line.strip()
(question_id, utterance, context, target_value, tokens, lemma_tokens,
pos_tags, ner_tags, ner_values, target_canon) = line.split("\t")
question = self.pre_process_sentence(tokens, ner_tags, ner_values)
target_canon = target_canon.split("|")
self.annotated_examples[question_id] = WikiExample(
question_id, question, target_canon, context)
self.annotated_tables[context] = []
counter += 1
print "Annotated examples loaded ", len(self.annotated_examples)
f.close()
def is_number_column(self, a):
for w in a:
if (len(w) != 1):
return False
if (not (is_number(w[0]))):
return False
return True
def convert_table(self, table):
answer = []
for i in range(len(table)):
temp = []
for j in range(len(table[i])):
temp.append(" ".join([str(w) for w in table[i][j]]))
answer.append(temp)
return answer
def load_annotated_tables(self):
for table in self.annotated_tables.keys():
annotated_table = table.replace("csv", "annotated")
orig_columns = []
processed_columns = []
f = tf.gfile.GFile(os.path.join(self.root_folder, annotated_table), "r")
counter = 0
for line in f:
if (counter > 0):
line = line.strip()
line = line + "\t" * (13 - len(line.split("\t")))
(row, col, read_id, content, tokens, lemma_tokens, pos_tags, ner_tags,
ner_values, number, date, num2, read_list) = line.split("\t")
counter += 1
f.close()
max_row = int(row)
max_col = int(col)
for i in range(max_col + 1):
orig_columns.append([])
processed_columns.append([])
for j in range(max_row + 1):
orig_columns[i].append(bad_number)
processed_columns[i].append(bad_number)
#print orig_columns
f = tf.gfile.GFile(os.path.join(self.root_folder, annotated_table), "r")
counter = 0
column_names = []
for line in f:
if (counter > 0):
line = line.strip()
line = line + "\t" * (13 - len(line.split("\t")))
(row, col, read_id, content, tokens, lemma_tokens, pos_tags, ner_tags,
ner_values, number, date, num2, read_list) = line.split("\t")
entry = self.pre_process_sentence(tokens, ner_tags, ner_values)
if (row == "-1"):
column_names.append(entry)
else:
orig_columns[int(col)][int(row)] = entry
if (len(entry) == 1 and is_number(entry[0])):
processed_columns[int(col)][int(row)] = float(entry[0])
else:
for single_entry in entry:
if (is_number(single_entry)):
processed_columns[int(col)][int(row)] = float(single_entry)
break
nt = ner_tags.split("|")
nv = ner_values.split("|")
for i_entry in range(len(tokens.split("|"))):
if (nt[i_entry] == "DATE" and
is_number(nv[i_entry].replace("-", "").replace("X", ""))):
processed_columns[int(col)][int(row)] = float(nv[
i_entry].replace("-", "").replace("X", ""))
#processed_columns[int(col)][int(row)] = float(nv[i_entry])
if (len(entry) == 1 and (is_number(entry[0]) or is_date(entry[0]) or
self.is_money(entry[0]))):
if (len(entry) == 1 and not (is_number(entry[0])) and
is_date(entry[0])):
entry[0] = entry[0].replace("X", "x")
counter += 1
word_columns = []
processed_word_columns = []
word_column_names = []
word_column_indices = []
number_columns = []
processed_number_columns = []
number_column_names = []
number_column_indices = []
for i in range(max_col + 1):
if (self.is_number_column(orig_columns[i])):
number_column_indices.append(i)
number_column_names.append(column_names[i])
temp = []
for w in orig_columns[i]:
if (is_number(w[0])):
temp.append(w[0])
number_columns.append(temp)
processed_number_columns.append(processed_columns[i])
else:
word_column_indices.append(i)
word_column_names.append(column_names[i])
word_columns.append(orig_columns[i])
processed_word_columns.append(processed_columns[i])
table_info = TableInfo(
word_columns, word_column_names, word_column_indices, number_columns,
number_column_names, number_column_indices, processed_word_columns,
processed_number_columns, orig_columns)
self.annotated_tables[table] = table_info
f.close()
def answer_classification(self):
lookup_questions = 0
number_lookup_questions = 0
word_lookup_questions = 0
ambiguous_lookup_questions = 0
number_questions = 0
bad_questions = 0
ice_bad_questions = 0
tot = 0
got = 0
ice = {}
with tf.gfile.GFile(
self.root_folder + "/arvind-with-norms-2.tsv", mode="r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if (not (self.annotated_examples.has_key(line.split("\t")[0]))):
continue
if (len(line.split("\t")) == 4):
line = line + "\t" * (5 - len(line.split("\t")))
if (not (is_number(line.split("\t")[2]))):
ice_bad_questions += 1
(example_id, ans_index, ans_raw, process_answer,
matched_cells) = line.split("\t")
if (ice.has_key(example_id)):
ice[example_id].append(line.split("\t"))
else:
ice[example_id] = [line.split("\t")]
for q_id in self.annotated_examples.keys():
tot += 1
example = self.annotated_examples[q_id]
table_info = self.annotated_tables[example.table_key]
# Figure out if the answer is numerical or lookup
n_cols = len(table_info.orig_columns)
n_rows = len(table_info.orig_columns[0])
example.lookup_matrix = np.zeros((n_rows, n_cols))
exact_matches = {}
for (example_id, ans_index, ans_raw, process_answer,
matched_cells) in ice[q_id]:
for match_cell in matched_cells.split("|"):
if (len(match_cell.split(",")) == 2):
(row, col) = match_cell.split(",")
row = int(row)
col = int(col)
if (row >= 0):
exact_matches[ans_index] = 1
answer_is_in_table = len(exact_matches) == len(example.answer)
if (answer_is_in_table):
for (example_id, ans_index, ans_raw, process_answer,
matched_cells) in ice[q_id]:
for match_cell in matched_cells.split("|"):
if (len(match_cell.split(",")) == 2):
(row, col) = match_cell.split(",")
row = int(row)
col = int(col)
example.lookup_matrix[row, col] = float(ans_index) + 1.0
example.lookup_number_answer = 0.0
if (answer_is_in_table):
lookup_questions += 1
if len(example.answer) == 1 and is_number(example.answer[0]):
example.number_answer = float(example.answer[0])
number_lookup_questions += 1
example.is_number_lookup = True
else:
#print "word lookup"
example.calc_answer = example.number_answer = 0.0
word_lookup_questions += 1
example.is_word_lookup = True
else:
if (len(example.answer) == 1 and is_number(example.answer[0])):
example.number_answer = example.answer[0]
example.is_number_calc = True
else:
bad_questions += 1
example.is_bad_example = True
example.is_unknown_answer = True
example.is_lookup = example.is_word_lookup or example.is_number_lookup
if not example.is_word_lookup and not example.is_bad_example:
number_questions += 1
example.calc_answer = example.answer[0]
example.lookup_number_answer = example.calc_answer
# Split up the lookup matrix into word part and number part
number_column_indices = table_info.number_column_indices
word_column_indices = table_info.word_column_indices
example.word_columns = table_info.word_columns
example.number_columns = table_info.number_columns
example.word_column_names = table_info.word_column_names
example.processed_number_columns = table_info.processed_number_columns
example.processed_word_columns = table_info.processed_word_columns
example.number_column_names = table_info.number_column_names
example.number_lookup_matrix = example.lookup_matrix[:,
number_column_indices]
example.word_lookup_matrix = example.lookup_matrix[:, word_column_indices]
def load(self):
train_data = []
dev_data = []
test_data = []
self.load_annotated_data(
os.path.join(self.data_folder, "training.annotated"))
self.load_annotated_tables()
self.answer_classification()
self.train_loader.load()
self.dev_loader.load()
for i in range(self.train_loader.num_questions()):
example = self.train_loader.examples[i]
example = self.annotated_examples[example]
train_data.append(example)
for i in range(self.dev_loader.num_questions()):
example = self.dev_loader.examples[i]
dev_data.append(self.annotated_examples[example])
self.load_annotated_data(
os.path.join(self.data_folder, "pristine-unseen-tables.annotated"))
self.load_annotated_tables()
self.answer_classification()
self.test_loader.load()
for i in range(self.test_loader.num_questions()):
example = self.test_loader.examples[i]
test_data.append(self.annotated_examples[example])
return train_data, dev_data, test_data
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment