Skip to content

GitLab

"git@developer.sourcefind.cn:Wenxuan/LightX2V.git" did not exist on "5e74be24683e1bd9cc7c219b1ef772953361ffc2"
Unverified Commit d5e826e3 authored by Steven Hickson's avatar Steven Hickson Committed by GitHub
Browse files

Merge branch 'master' into master

parents e1ac09e1 fc37f117
Hide whitespace changes
Inline Side-by-side
Showing with 114 additions and 90 deletions
research/neural_gpu/data_utils.py
View file @ d5e826e3
...@@ -21,6 +21,7 @@ import sys ...@@ -21,6 +21,7 @@ import sys
import time import time
import numpy as np import numpy as np
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
import program_utils import program_utils
......
research/neural_gpu/neural_gpu.py
View file @ d5e826e3
...@@ -17,6 +17,7 @@ ...@@ -17,6 +17,7 @@
import time import time
import numpy as np import numpy as np
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
from tensorflow.python.framework import function from tensorflow.python.framework import function
...@@ -500,8 +501,10 @@ class NeuralGPU(object): ...@@ -500,8 +501,10 @@ class NeuralGPU(object):
return tf.reduce_sum(encoder_outputs * tf.expand_dims(mask, 2), 1) return tf.reduce_sum(encoder_outputs * tf.expand_dims(mask, 2), 1)
with tf.variable_scope("decoder"): with tf.variable_scope("decoder"):
def decoder_loop_fn((state, prev_cell_out, _), (cell_inp, cur_tgt)): def decoder_loop_fn(state__prev_cell_out__unused, cell_inp__cur_tgt):
"""Decoder loop function.""" """Decoder loop function."""
state, prev_cell_out, _ = state__prev_cell_out__unused
cell_inp, cur_tgt = cell_inp__cur_tgt
attn_q = tf.layers.dense(prev_cell_out, height * nmaps, attn_q = tf.layers.dense(prev_cell_out, height * nmaps,
name="attn_query") name="attn_query")
attn_res = attention_query(attn_q, tf.get_variable( attn_res = attention_query(attn_q, tf.get_variable(
......
research/neural_gpu/neural_gpu_trainer.py
View file @ d5e826e3
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
# ============================================================================== # ==============================================================================
"""Neural GPU.""" """Neural GPU."""
from __future__ import print_function
import math import math
import os import os
import random import random
...@@ -22,6 +24,7 @@ import threading ...@@ -22,6 +24,7 @@ import threading
import time import time
import numpy as np import numpy as np
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
import program_utils import program_utils
...@@ -144,7 +147,7 @@ def read_data(source_path, target_path, buckets, max_size=None, print_out=True): ...@@ -144,7 +147,7 @@ def read_data(source_path, target_path, buckets, max_size=None, print_out=True):
while source and target and (not max_size or counter < max_size): while source and target and (not max_size or counter < max_size):
counter += 1 counter += 1
if counter % 100000 == 0 and print_out: if counter % 100000 == 0 and print_out:
print " reading data line %d" % counter print(" reading data line %d" % counter)
sys.stdout.flush() sys.stdout.flush()
source_ids = [int(x) for x in source.split()] source_ids = [int(x) for x in source.split()]
target_ids = [int(x) for x in target.split()] target_ids = [int(x) for x in target.split()]
...@@ -188,7 +191,7 @@ def read_data_into_global(source_path, target_path, buckets, ...@@ -188,7 +191,7 @@ def read_data_into_global(source_path, target_path, buckets,
global_train_set["wmt"].append(data_set) global_train_set["wmt"].append(data_set)
train_total_size = calculate_buckets_scale(data_set, buckets, "wmt") train_total_size = calculate_buckets_scale(data_set, buckets, "wmt")
if print_out: if print_out:
print " Finished global data reading (%d)." % train_total_size print(" Finished global data reading (%d)." % train_total_size)
def initialize(sess=None): def initialize(sess=None):
...@@ -552,7 +555,7 @@ def score_beams_prog(beams, target, inp, history, print_out=False, ...@@ -552,7 +555,7 @@ def score_beams_prog(beams, target, inp, history, print_out=False,
for h in history] for h in history]
tgt_set = set(target) tgt_set = set(target)
if print_out: if print_out:
print "target: ", tgt_prog print("target: ", tgt_prog)
inps, tgt_outs = [], [] inps, tgt_outs = [], []
for i in xrange(3): for i in xrange(3):
ilist = [inp[i + 1, l] for l in xrange(inp.shape[1])] ilist = [inp[i + 1, l] for l in xrange(inp.shape[1])]
...@@ -566,11 +569,11 @@ def score_beams_prog(beams, target, inp, history, print_out=False, ...@@ -566,11 +569,11 @@ def score_beams_prog(beams, target, inp, history, print_out=False,
if len(olist) == 1: if len(olist) == 1:
tgt_outs.append(olist[0]) tgt_outs.append(olist[0])
else: else:
print [program_utils.prog_vocab[x] for x in ilist if x > 0] print([program_utils.prog_vocab[x] for x in ilist if x > 0])
print olist print(olist)
print tgt_prog print(tgt_prog)
print program_utils.evaluate(tgt_prog, {"a": inps[-1]}) print(program_utils.evaluate(tgt_prog, {"a": inps[-1]}))
print "AAAAA" print("AAAAA")
tgt_outs.append(olist[0]) tgt_outs.append(olist[0])
if not test_mode: if not test_mode:
for _ in xrange(7): for _ in xrange(7):
...@@ -602,7 +605,7 @@ def score_beams_prog(beams, target, inp, history, print_out=False, ...@@ -602,7 +605,7 @@ def score_beams_prog(beams, target, inp, history, print_out=False,
best_prog = b_prog best_prog = b_prog
best_score = score best_score = score
if print_out: if print_out:
print "best score: ", best_score, " best prog: ", best_prog print("best score: ", best_score, " best prog: ", best_prog)
return best, best_score return best, best_score
...@@ -719,7 +722,7 @@ def train(): ...@@ -719,7 +722,7 @@ def train():
inp = new_inp inp = new_inp
# If all results are great, stop (todo: not to wait for all?). # If all results are great, stop (todo: not to wait for all?).
if FLAGS.nprint > 1: if FLAGS.nprint > 1:
print scores print(scores)
if sum(scores) / float(len(scores)) >= 10.0: if sum(scores) / float(len(scores)) >= 10.0:
break break
# The final step with the true target. # The final step with the true target.
...@@ -735,7 +738,7 @@ def train(): ...@@ -735,7 +738,7 @@ def train():
errors, total, seq_err = data.accuracy( errors, total, seq_err = data.accuracy(
inp, res, target, batch_size, 0, new_target, scores) inp, res, target, batch_size, 0, new_target, scores)
if FLAGS.nprint > 1: if FLAGS.nprint > 1:
print "seq_err: ", seq_err print("seq_err: ", seq_err)
acc_total += total acc_total += total
acc_errors += errors acc_errors += errors
acc_seq_err += seq_err acc_seq_err += seq_err
...@@ -944,8 +947,8 @@ def interactive(): ...@@ -944,8 +947,8 @@ def interactive():
for v in tf.trainable_variables(): for v in tf.trainable_variables():
shape = v.get_shape().as_list() shape = v.get_shape().as_list()
total += mul(shape) total += mul(shape)
print (v.name, shape, mul(shape)) print(v.name, shape, mul(shape))
print total print(total)
# Start interactive loop. # Start interactive loop.
sys.stdout.write("Input to Neural GPU Translation Model.\n") sys.stdout.write("Input to Neural GPU Translation Model.\n")
sys.stdout.write("> ") sys.stdout.write("> ")
...@@ -960,7 +963,7 @@ def interactive(): ...@@ -960,7 +963,7 @@ def interactive():
normalize_digits=FLAGS.normalize_digits) normalize_digits=FLAGS.normalize_digits)
else: else:
token_ids = wmt.sentence_to_token_ids(inpt, en_vocab) token_ids = wmt.sentence_to_token_ids(inpt, en_vocab)
print [rev_en_vocab[t] for t in token_ids] print([rev_en_vocab[t] for t in token_ids])
# Which bucket does it belong to? # Which bucket does it belong to?
buckets = [b for b in xrange(len(data.bins)) buckets = [b for b in xrange(len(data.bins))
if data.bins[b] >= max(len(token_ids), len(cures))] if data.bins[b] >= max(len(token_ids), len(cures))]
...@@ -986,12 +989,12 @@ def interactive(): ...@@ -986,12 +989,12 @@ def interactive():
loss = loss[0] - (data.bins[bucket_id] * FLAGS.length_norm) loss = loss[0] - (data.bins[bucket_id] * FLAGS.length_norm)
outputs = [int(np.argmax(logit, axis=1)) outputs = [int(np.argmax(logit, axis=1))
for logit in output_logits] for logit in output_logits]
print [rev_fr_vocab[t] for t in outputs] print([rev_fr_vocab[t] for t in outputs])
print loss, data.bins[bucket_id] print(loss, data.bins[bucket_id])
print linearize(outputs, rev_fr_vocab) print(linearize(outputs, rev_fr_vocab))
cures.append(outputs[gen_idx]) cures.append(outputs[gen_idx])
print cures print(cures)
print linearize(cures, rev_fr_vocab) print(linearize(cures, rev_fr_vocab))
if FLAGS.simple_tokenizer: if FLAGS.simple_tokenizer:
cur_out = outputs cur_out = outputs
if wmt.EOS_ID in cur_out: if wmt.EOS_ID in cur_out:
...@@ -1002,11 +1005,11 @@ def interactive(): ...@@ -1002,11 +1005,11 @@ def interactive():
if loss < result_cost: if loss < result_cost:
result = outputs result = outputs
result_cost = loss result_cost = loss
print ("FINAL", result_cost) print("FINAL", result_cost)
print [rev_fr_vocab[t] for t in result] print([rev_fr_vocab[t] for t in result])
print linearize(result, rev_fr_vocab) print(linearize(result, rev_fr_vocab))
else: else:
print "TOOO_LONG" print("TOOO_LONG")
sys.stdout.write("> ") sys.stdout.write("> ")
sys.stdout.flush() sys.stdout.flush()
inpt = sys.stdin.readline(), "" inpt = sys.stdin.readline(), ""
......
research/neural_gpu/program_utils.py
View file @ d5e826e3
...@@ -31,7 +31,7 @@ class ListType(object): ...@@ -31,7 +31,7 @@ class ListType(object):
if not isinstance(other, ListType): if not isinstance(other, ListType):
return False return False
return self.arg == other.arg return self.arg == other.arg
def __hash__(self): def __hash__(self):
return hash(self.arg) return hash(self.arg)
...@@ -68,7 +68,7 @@ class FunctionType(object): ...@@ -68,7 +68,7 @@ class FunctionType(object):
class Function(object): class Function(object):
def __init__(self, name, arg_types, output_type, fn_arg_types = None): def __init__(self, name, arg_types, output_type, fn_arg_types = None):
self.name = name self.name = name
self.arg_types = arg_types self.arg_types = arg_types
self.fn_arg_types = fn_arg_types or [] self.fn_arg_types = fn_arg_types or []
self.output_type = output_type self.output_type = output_type
...@@ -113,14 +113,14 @@ def minus_one(x): return x - 1 ...@@ -113,14 +113,14 @@ def minus_one(x): return x - 1
def times_two(x): return x * 2 def times_two(x): return x * 2
def neg(x): return x * (-1) def neg(x): return x * (-1)
def div_two(x): return int(x/2) def div_two(x): return int(x/2)
def sq(x): return x**2 def sq(x): return x**2
def times_three(x): return x * 3 def times_three(x): return x * 3
def div_three(x): return int(x/3) def div_three(x): return int(x/3)
def times_four(x): return x * 4 def times_four(x): return x * 4
def div_four(x): return int(x/4) def div_four(x): return int(x/4)
# Int -> Bool # Int -> Bool
def pos(x): return x > 0 def pos(x): return x > 0
def neg(x): return x < 0 def neg(x): return x < 0
def even(x): return x%2 == 0 def even(x): return x%2 == 0
def odd(x): return x%2 == 1 def odd(x): return x%2 == 1
...@@ -131,24 +131,24 @@ def sub(x, y): return x - y ...@@ -131,24 +131,24 @@ def sub(x, y): return x - y
def mul(x, y): return x * y def mul(x, y): return x * y
# HOFs # HOFs
f_map = Function("map", [ListType("Int")], f_map = Function("map", [ListType("Int")],
ListType("Int"), ListType("Int"),
[FunctionType(["Int", "Int"])]) [FunctionType(["Int", "Int"])])
f_filter = Function("filter", [ListType("Int")], f_filter = Function("filter", [ListType("Int")],
ListType("Int"), ListType("Int"),
[FunctionType(["Int", "Bool"])]) [FunctionType(["Int", "Bool"])])
f_count = Function("c_count", [ListType("Int")], f_count = Function("c_count", [ListType("Int")],
"Int", "Int",
[FunctionType(["Int", "Bool"])]) [FunctionType(["Int", "Bool"])])
def c_count(f, xs): return len([x for x in xs if f(x)]) def c_count(f, xs): return len([x for x in xs if f(x)])
f_zipwith = Function("c_zipwith", [ListType("Int"), ListType("Int")], f_zipwith = Function("c_zipwith", [ListType("Int"), ListType("Int")],
ListType("Int"), ListType("Int"),
[FunctionType(["Int", "Int", "Int"])]) #FIX [FunctionType(["Int", "Int", "Int"])]) #FIX
def c_zipwith(f, xs, ys): return [f(x, y) for (x, y) in zip(xs, ys)] def c_zipwith(f, xs, ys): return [f(x, y) for (x, y) in zip(xs, ys)]
f_scan = Function("c_scan", [ListType("Int")], f_scan = Function("c_scan", [ListType("Int")],
ListType("Int"), ListType("Int"),
[FunctionType(["Int", "Int", "Int"])]) [FunctionType(["Int", "Int", "Int"])])
def c_scan(f, xs): def c_scan(f, xs):
out = xs out = xs
...@@ -177,7 +177,7 @@ def evaluate(program_str, input_names_to_vals, default="ERROR"): ...@@ -177,7 +177,7 @@ def evaluate(program_str, input_names_to_vals, default="ERROR"):
with stdoutIO() as s: with stdoutIO() as s:
# pylint: disable=bare-except # pylint: disable=bare-except
try: try:
exec exec_str + " print(out)" exec(exec_str + " print(out)")
return s.getvalue()[:-1] return s.getvalue()[:-1]
except: except:
return default return default
...@@ -186,7 +186,7 @@ def evaluate(program_str, input_names_to_vals, default="ERROR"): ...@@ -186,7 +186,7 @@ def evaluate(program_str, input_names_to_vals, default="ERROR"):
class Statement(object): class Statement(object):
"""Statement class.""" """Statement class."""
def __init__(self, fn, output_var, arg_vars, fn_args=None): def __init__(self, fn, output_var, arg_vars, fn_args=None):
self.fn = fn self.fn = fn
self.output_var = output_var self.output_var = output_var
...@@ -290,7 +290,7 @@ class Program(object): ...@@ -290,7 +290,7 @@ class Program(object):
with stdoutIO() as s: with stdoutIO() as s:
# pylint: disable=exec-used # pylint: disable=exec-used
exec inp_str + self.body + "; print(out)" exec(inp_str + self.body + "; print(out)")
# pylint: enable=exec-used # pylint: enable=exec-used
return s.getvalue()[:-1] return s.getvalue()[:-1]
...@@ -412,11 +412,11 @@ def gen(max_len, how_many): ...@@ -412,11 +412,11 @@ def gen(max_len, how_many):
else: else:
outcomes_to_programs[outcome_str] = t.flat_str() outcomes_to_programs[outcome_str] = t.flat_str()
if counter % 5000 == 0: if counter % 5000 == 0:
print "== proggen: tried: " + str(counter) print("== proggen: tried: " + str(counter))
print "== proggen: kept: " + str(len(outcomes_to_programs)) print("== proggen: kept: " + str(len(outcomes_to_programs)))
if counter % 250000 == 0 and save_prefix is not None: if counter % 250000 == 0 and save_prefix is not None:
print "saving..." print("saving...")
save_counter = 0 save_counter = 0
progfilename = os.path.join(save_prefix, "prog_" + str(counter) + ".txt") progfilename = os.path.join(save_prefix, "prog_" + str(counter) + ".txt")
iofilename = os.path.join(save_prefix, "io_" + str(counter) + ".txt") iofilename = os.path.join(save_prefix, "io_" + str(counter) + ".txt")
...@@ -431,7 +431,7 @@ def gen(max_len, how_many): ...@@ -431,7 +431,7 @@ def gen(max_len, how_many):
for (o, p) in outcomes_to_programs.iteritems(): for (o, p) in outcomes_to_programs.iteritems():
save_counter += 1 save_counter += 1
if save_counter % 500 == 0: if save_counter % 500 == 0:
print "saving %d of %d" % (save_counter, len(outcomes_to_programs)) print("saving %d of %d" % (save_counter, len(outcomes_to_programs)))
fp.write(p+"\n") fp.write(p+"\n")
fi.write(o+"\n") fi.write(o+"\n")
ftp.write(str(tokenize(p, tokens))+"\n") ftp.write(str(tokenize(p, tokens))+"\n")
......
research/neural_gpu/wmt_utils.py
View file @ d5e826e3
...@@ -14,6 +14,8 @@ ...@@ -14,6 +14,8 @@
# ============================================================================== # ==============================================================================
"""Utilities for downloading data from WMT, tokenizing, vocabularies.""" """Utilities for downloading data from WMT, tokenizing, vocabularies."""
from __future__ import print_function
import gzip import gzip
import os import os
import re import re
...@@ -53,20 +55,20 @@ _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/dev-v2.tgz" ...@@ -53,20 +55,20 @@ _WMT_ENFR_DEV_URL = "http://www.statmt.org/wmt15/dev-v2.tgz"
def maybe_download(directory, filename, url): def maybe_download(directory, filename, url):
"""Download filename from url unless it's already in directory.""" """Download filename from url unless it's already in directory."""
if not tf.gfile.Exists(directory): if not tf.gfile.Exists(directory):
print "Creating directory %s" % directory print("Creating directory %s" % directory)
os.mkdir(directory) os.mkdir(directory)
filepath = os.path.join(directory, filename) filepath = os.path.join(directory, filename)
if not tf.gfile.Exists(filepath): if not tf.gfile.Exists(filepath):
print "Downloading %s to %s" % (url, filepath) print("Downloading %s to %s" % (url, filepath))
filepath, _ = urllib.request.urlretrieve(url, filepath) filepath, _ = urllib.request.urlretrieve(url, filepath)
statinfo = os.stat(filepath) statinfo = os.stat(filepath)
print "Successfully downloaded", filename, statinfo.st_size, "bytes" print("Successfully downloaded", filename, statinfo.st_size, "bytes")
return filepath return filepath
def gunzip_file(gz_path, new_path): def gunzip_file(gz_path, new_path):
"""Unzips from gz_path into new_path.""" """Unzips from gz_path into new_path."""
print "Unpacking %s to %s" % (gz_path, new_path) print("Unpacking %s to %s" % (gz_path, new_path))
with gzip.open(gz_path, "rb") as gz_file: with gzip.open(gz_path, "rb") as gz_file:
with open(new_path, "wb") as new_file: with open(new_path, "wb") as new_file:
for line in gz_file: for line in gz_file:
...@@ -80,7 +82,7 @@ def get_wmt_enfr_train_set(directory): ...@@ -80,7 +82,7 @@ def get_wmt_enfr_train_set(directory):
tf.gfile.Exists(train_path +".en")): tf.gfile.Exists(train_path +".en")):
corpus_file = maybe_download(directory, "training-giga-fren.tar", corpus_file = maybe_download(directory, "training-giga-fren.tar",
_WMT_ENFR_TRAIN_URL) _WMT_ENFR_TRAIN_URL)
print "Extracting tar file %s" % corpus_file print("Extracting tar file %s" % corpus_file)
with tarfile.open(corpus_file, "r") as corpus_tar: with tarfile.open(corpus_file, "r") as corpus_tar:
corpus_tar.extractall(directory) corpus_tar.extractall(directory)
gunzip_file(train_path + ".fr.gz", train_path + ".fr") gunzip_file(train_path + ".fr.gz", train_path + ".fr")
...@@ -95,7 +97,7 @@ def get_wmt_enfr_dev_set(directory): ...@@ -95,7 +97,7 @@ def get_wmt_enfr_dev_set(directory):
if not (tf.gfile.Exists(dev_path + ".fr") and if not (tf.gfile.Exists(dev_path + ".fr") and
tf.gfile.Exists(dev_path + ".en")): tf.gfile.Exists(dev_path + ".en")):
dev_file = maybe_download(directory, "dev-v2.tgz", _WMT_ENFR_DEV_URL) dev_file = maybe_download(directory, "dev-v2.tgz", _WMT_ENFR_DEV_URL)
print "Extracting tgz file %s" % dev_file print("Extracting tgz file %s" % dev_file)
with tarfile.open(dev_file, "r:gz") as dev_tar: with tarfile.open(dev_file, "r:gz") as dev_tar:
fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr") fr_dev_file = dev_tar.getmember("dev/" + dev_name + ".fr")
en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en") en_dev_file = dev_tar.getmember("dev/" + dev_name + ".en")
...@@ -206,7 +208,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size, ...@@ -206,7 +208,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
normalize_digits: Boolean; if true, all digits are replaced by 0s. normalize_digits: Boolean; if true, all digits are replaced by 0s.
""" """
if not tf.gfile.Exists(vocabulary_path): if not tf.gfile.Exists(vocabulary_path):
print "Creating vocabulary %s from data %s" % (vocabulary_path, data_path) print("Creating vocabulary %s from data %s" % (vocabulary_path, data_path))
vocab, chars = {}, {} vocab, chars = {}, {}
for c in _PUNCTUATION: for c in _PUNCTUATION:
chars[c] = 1 chars[c] = 1
...@@ -218,7 +220,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size, ...@@ -218,7 +220,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
line = " ".join(line_in.split()) line = " ".join(line_in.split())
counter += 1 counter += 1
if counter % 100000 == 0: if counter % 100000 == 0:
print " processing fr line %d" % counter print(" processing fr line %d" % counter)
for c in line: for c in line:
if c in chars: if c in chars:
chars[c] += 1 chars[c] += 1
...@@ -240,7 +242,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size, ...@@ -240,7 +242,7 @@ def create_vocabulary(vocabulary_path, data_path, max_vocabulary_size,
line = " ".join(line_in.split()) line = " ".join(line_in.split())
counter += 1 counter += 1
if counter % 100000 == 0: if counter % 100000 == 0:
print " processing en line %d" % counter print(" processing en line %d" % counter)
for c in line: for c in line:
if c in chars: if c in chars:
chars[c] += 1 chars[c] += 1
...@@ -371,7 +373,7 @@ def data_to_token_ids(data_path, target_path, vocabulary_path, ...@@ -371,7 +373,7 @@ def data_to_token_ids(data_path, target_path, vocabulary_path,
normalize_digits: Boolean; if true, all digits are replaced by 0s. normalize_digits: Boolean; if true, all digits are replaced by 0s.
""" """
if not tf.gfile.Exists(target_path): if not tf.gfile.Exists(target_path):
print "Tokenizing data in %s" % data_path print("Tokenizing data in %s" % data_path)
vocab, _ = initialize_vocabulary(vocabulary_path) vocab, _ = initialize_vocabulary(vocabulary_path)
with tf.gfile.GFile(data_path, mode="rb") as data_file: with tf.gfile.GFile(data_path, mode="rb") as data_file:
with tf.gfile.GFile(target_path, mode="w") as tokens_file: with tf.gfile.GFile(target_path, mode="w") as tokens_file:
...@@ -379,7 +381,7 @@ def data_to_token_ids(data_path, target_path, vocabulary_path, ...@@ -379,7 +381,7 @@ def data_to_token_ids(data_path, target_path, vocabulary_path,
for line in data_file: for line in data_file:
counter += 1 counter += 1
if counter % 100000 == 0: if counter % 100000 == 0:
print " tokenizing line %d" % counter print(" tokenizing line %d" % counter)
token_ids = sentence_to_token_ids(line, vocab, tokenizer, token_ids = sentence_to_token_ids(line, vocab, tokenizer,
normalize_digits) normalize_digits)
tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n") tokens_file.write(" ".join([str(tok) for tok in token_ids]) + "\n")
......
research/neural_programmer/data_utils.py
View file @ d5e826e3
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
"""Functions for constructing vocabulary, converting the examples to integer format and building the required masks for batch computation Author: aneelakantan (Arvind Neelakantan) """Functions for constructing vocabulary, converting the examples to integer format and building the required masks for batch computation Author: aneelakantan (Arvind Neelakantan)
""" """
from __future__ import print_function
import copy import copy
import numbers import numbers
import numpy as np import numpy as np
...@@ -536,15 +538,15 @@ def add_special_words(utility): ...@@ -536,15 +538,15 @@ def add_special_words(utility):
utility.reverse_word_ids[utility.word_ids[ utility.reverse_word_ids[utility.word_ids[
utility.entry_match_token]] = utility.entry_match_token utility.entry_match_token]] = utility.entry_match_token
utility.entry_match_token_id = utility.word_ids[utility.entry_match_token] utility.entry_match_token_id = utility.word_ids[utility.entry_match_token]
print "entry match token: ", utility.word_ids[ print("entry match token: ", utility.word_ids[
utility.entry_match_token], utility.entry_match_token_id utility.entry_match_token], utility.entry_match_token_id)
utility.words.append(utility.column_match_token) utility.words.append(utility.column_match_token)
utility.word_ids[utility.column_match_token] = len(utility.word_ids) utility.word_ids[utility.column_match_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[ utility.reverse_word_ids[utility.word_ids[
utility.column_match_token]] = utility.column_match_token utility.column_match_token]] = utility.column_match_token
utility.column_match_token_id = utility.word_ids[utility.column_match_token] utility.column_match_token_id = utility.word_ids[utility.column_match_token]
print "entry match token: ", utility.word_ids[ print("entry match token: ", utility.word_ids[
utility.column_match_token], utility.column_match_token_id utility.column_match_token], utility.column_match_token_id)
utility.words.append(utility.dummy_token) utility.words.append(utility.dummy_token)
utility.word_ids[utility.dummy_token] = len(utility.word_ids) utility.word_ids[utility.dummy_token] = len(utility.word_ids)
utility.reverse_word_ids[utility.word_ids[ utility.reverse_word_ids[utility.word_ids[
......
research/neural_programmer/model.py
View file @ d5e826e3
...@@ -15,6 +15,8 @@ ...@@ -15,6 +15,8 @@
"""Author: aneelakantan (Arvind Neelakantan) """Author: aneelakantan (Arvind Neelakantan)
""" """
from __future__ import print_function
import numpy as np import numpy as np
import tensorflow as tf import tensorflow as tf
import nn_utils import nn_utils
...@@ -545,7 +547,7 @@ class Graph(): ...@@ -545,7 +547,7 @@ class Graph():
self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type) self.batch_log_prob = tf.zeros([self.batch_size], dtype=self.data_type)
#Perform max_passes and at each pass select operation and column #Perform max_passes and at each pass select operation and column
for curr_pass in range(max_passes): for curr_pass in range(max_passes):
print "step: ", curr_pass print("step: ", curr_pass)
output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass( output, select, softmax, soft_softmax, column_softmax, soft_column_softmax = self.one_pass(
select, question_embedding, hidden_vectors, hprev, prev_select_1, select, question_embedding, hidden_vectors, hprev, prev_select_1,
curr_pass) curr_pass)
...@@ -633,10 +635,10 @@ class Graph(): ...@@ -633,10 +635,10 @@ class Graph():
self.params = params self.params = params
batch_size = self.batch_size batch_size = self.batch_size
learning_rate = tf.cast(self.utility.FLAGS.learning_rate, self.data_type) learning_rate = tf.cast(self.utility.FLAGS.learning_rate, self.data_type)
self.total_cost = self.compute_error() self.total_cost = self.compute_error()
optimize_params = self.params.values() optimize_params = self.params.values()
optimize_names = self.params.keys() optimize_names = self.params.keys()
print "optimize params ", optimize_names print("optimize params ", optimize_names)
if (self.utility.FLAGS.l2_regularizer > 0.0): if (self.utility.FLAGS.l2_regularizer > 0.0):
reg_cost = 0.0 reg_cost = 0.0
for ind_param in self.params.keys(): for ind_param in self.params.keys():
...@@ -645,7 +647,7 @@ class Graph(): ...@@ -645,7 +647,7 @@ class Graph():
grads = tf.gradients(self.total_cost, optimize_params, name="gradients") grads = tf.gradients(self.total_cost, optimize_params, name="gradients")
grad_norm = 0.0 grad_norm = 0.0
for p, name in zip(grads, optimize_names): for p, name in zip(grads, optimize_names):
print "grads: ", p, name print("grads: ", p, name)
if isinstance(p, tf.IndexedSlices): if isinstance(p, tf.IndexedSlices):
grad_norm += tf.reduce_sum(p.values * p.values) grad_norm += tf.reduce_sum(p.values * p.values)
elif not (p == None): elif not (p == None):
...@@ -672,7 +674,6 @@ class Graph(): ...@@ -672,7 +674,6 @@ class Graph():
learning_rate, learning_rate,
epsilon=tf.cast(self.utility.FLAGS.eps, self.data_type), epsilon=tf.cast(self.utility.FLAGS.eps, self.data_type),
use_locking=True) use_locking=True)
self.step = adam.apply_gradients(zip(grads, optimize_params), self.step = adam.apply_gradients(zip(grads, optimize_params),
global_step=self.global_step) global_step=self.global_step)
self.init_op = tf.global_variables_initializer() self.init_op = tf.global_variables_initializer()
research/neural_programmer/neural_programmer.py
View file @ d5e826e3
...@@ -18,6 +18,8 @@ This file calls functions to load & pre-process data, construct the TF graph ...@@ -18,6 +18,8 @@ This file calls functions to load & pre-process data, construct the TF graph
and performs training or evaluation as specified by the flag evaluator_job and performs training or evaluation as specified by the flag evaluator_job
Author: aneelakantan (Arvind Neelakantan) Author: aneelakantan (Arvind Neelakantan)
""" """
from __future__ import print_function
import time import time
from random import Random from random import Random
import numpy as np import numpy as np
...@@ -113,9 +115,9 @@ def evaluate(sess, data, batch_size, graph, i): ...@@ -113,9 +115,9 @@ def evaluate(sess, data, batch_size, graph, i):
graph)) graph))
gc += ct * batch_size gc += ct * batch_size
num_examples += batch_size num_examples += batch_size
print "dev set accuracy after ", i, " : ", gc / num_examples print("dev set accuracy after ", i, " : ", gc / num_examples)
print num_examples, len(data) print(num_examples, len(data))
print "--------" print("--------")
def Train(graph, utility, batch_size, train_data, sess, model_dir, def Train(graph, utility, batch_size, train_data, sess, model_dir,
...@@ -142,15 +144,15 @@ def Train(graph, utility, batch_size, train_data, sess, model_dir, ...@@ -142,15 +144,15 @@ def Train(graph, utility, batch_size, train_data, sess, model_dir,
if (i > 0 and i % FLAGS.eval_cycle == 0): if (i > 0 and i % FLAGS.eval_cycle == 0):
end = time.time() end = time.time()
time_taken = end - start time_taken = end - start
print "step ", i, " ", time_taken, " seconds " print("step ", i, " ", time_taken, " seconds ")
start = end start = end
print " printing train set loss: ", train_set_loss / utility.FLAGS.eval_cycle print(" printing train set loss: ", train_set_loss / utility.FLAGS.eval_cycle)
train_set_loss = 0.0 train_set_loss = 0.0
def master(train_data, dev_data, utility): def master(train_data, dev_data, utility):
#creates TF graph and calls trainer or evaluator #creates TF graph and calls trainer or evaluator
batch_size = utility.FLAGS.batch_size batch_size = utility.FLAGS.batch_size
model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/" model_dir = utility.FLAGS.output_dir + "/model" + utility.FLAGS.job_id + "/"
#create all paramters of the model #create all paramters of the model
param_class = parameters.Parameters(utility) param_class = parameters.Parameters(utility)
...@@ -183,23 +185,23 @@ def master(train_data, dev_data, utility): ...@@ -183,23 +185,23 @@ def master(train_data, dev_data, utility):
file_list = sorted(selected_models.items(), key=lambda x: x[0]) file_list = sorted(selected_models.items(), key=lambda x: x[0])
if (len(file_list) > 0): if (len(file_list) > 0):
file_list = file_list[0:len(file_list) - 1] file_list = file_list[0:len(file_list) - 1]
print "list of models: ", file_list print("list of models: ", file_list)
for model_file in file_list: for model_file in file_list:
model_file = model_file[1] model_file = model_file[1]
print "restoring: ", model_file print("restoring: ", model_file)
saver.restore(sess, model_dir + "/" + model_file) saver.restore(sess, model_dir + "/" + model_file)
model_step = int( model_step = int(
model_file.split("_")[len(model_file.split("_")) - 1]) model_file.split("_")[len(model_file.split("_")) - 1])
print "evaluating on dev ", model_file, model_step print("evaluating on dev ", model_file, model_step)
evaluate(sess, dev_data, batch_size, graph, model_step) evaluate(sess, dev_data, batch_size, graph, model_step)
else: else:
ckpt = tf.train.get_checkpoint_state(model_dir) ckpt = tf.train.get_checkpoint_state(model_dir)
print "model dir: ", model_dir print("model dir: ", model_dir)
if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))): if (not (tf.gfile.IsDirectory(utility.FLAGS.output_dir))):
print "create dir: ", utility.FLAGS.output_dir print("create dir: ", utility.FLAGS.output_dir)
tf.gfile.MkDir(utility.FLAGS.output_dir) tf.gfile.MkDir(utility.FLAGS.output_dir)
if (not (tf.gfile.IsDirectory(model_dir))): if (not (tf.gfile.IsDirectory(model_dir))):
print "create dir: ", model_dir print("create dir: ", model_dir)
tf.gfile.MkDir(model_dir) tf.gfile.MkDir(model_dir)
Train(graph, utility, batch_size, train_data, sess, model_dir, Train(graph, utility, batch_size, train_data, sess, model_dir,
saver) saver)
...@@ -225,10 +227,10 @@ def main(args): ...@@ -225,10 +227,10 @@ def main(args):
train_data = data_utils.complete_wiki_processing(train_data, utility, True) train_data = data_utils.complete_wiki_processing(train_data, utility, True)
dev_data = data_utils.complete_wiki_processing(dev_data, utility, False) dev_data = data_utils.complete_wiki_processing(dev_data, utility, False)
test_data = data_utils.complete_wiki_processing(test_data, utility, False) test_data = data_utils.complete_wiki_processing(test_data, utility, False)
print "# train examples ", len(train_data) print("# train examples ", len(train_data))
print "# dev examples ", len(dev_data) print("# dev examples ", len(dev_data))
print "# test examples ", len(test_data) print("# test examples ", len(test_data))
print "running open source" print("running open source")
#construct TF graph and train or evaluate #construct TF graph and train or evaluate
master(train_data, dev_data, utility) master(train_data, dev_data, utility)
......
research/neural_programmer/parameters.py
View file @ d5e826e3
...@@ -59,7 +59,7 @@ class Parameters: ...@@ -59,7 +59,7 @@ class Parameters:
#Biases for the gates and cell #Biases for the gates and cell
for bias in ["i", "f", "c", "o"]: for bias in ["i", "f", "c", "o"]:
if (bias == "f"): if (bias == "f"):
print "forget gate bias" print("forget gate bias")
params[key + "_" + bias] = tf.Variable( params[key + "_" + bias] = tf.Variable(
tf.random_uniform([embedding_dims], 1.0, 1.1, self.utility. tf.random_uniform([embedding_dims], 1.0, 1.1, self.utility.
tf_data_type[self.utility.FLAGS.data_type])) tf_data_type[self.utility.FLAGS.data_type]))
......
research/neural_programmer/wiki_data.py
View file @ d5e826e3
...@@ -22,6 +22,8 @@ columns. ...@@ -22,6 +22,8 @@ columns.
lookup answer (or matrix) is also split into number and word lookup matrix lookup answer (or matrix) is also split into number and word lookup matrix
Author: aneelakantan (Arvind Neelakantan) Author: aneelakantan (Arvind Neelakantan)
""" """
from __future__ import print_function
import math import math
import os import os
import re import re
...@@ -56,7 +58,7 @@ def correct_unicode(string): ...@@ -56,7 +58,7 @@ def correct_unicode(string):
#string = re.sub("[“”«»]", "\"", string) #string = re.sub("[“”«»]", "\"", string)
#string = re.sub("[•†‡]", "", string) #string = re.sub("[•†‡]", "", string)
#string = re.sub("[‐‑–—]", "-", string) #string = re.sub("[‐‑–—]", "-", string)
string = re.sub(ur'[\u2E00-\uFFFF]', "", string) string = re.sub(r'[\u2E00-\uFFFF]', "", string)
string = re.sub("\\s+", " ", string).strip() string = re.sub("\\s+", " ", string).strip()
return string return string
...@@ -78,7 +80,7 @@ def full_normalize(string): ...@@ -78,7 +80,7 @@ def full_normalize(string):
# Remove trailing info in brackets # Remove trailing info in brackets
string = re.sub("\[[^\]]*\]", "", string) string = re.sub("\[[^\]]*\]", "", string)
# Remove most unicode characters in other languages # Remove most unicode characters in other languages
string = re.sub(ur'[\u007F-\uFFFF]', "", string.strip()) string = re.sub(r'[\u007F-\uFFFF]', "", string.strip())
# Remove trailing info in parenthesis # Remove trailing info in parenthesis
string = re.sub("\([^)]*\)$", "", string.strip()) string = re.sub("\([^)]*\)$", "", string.strip())
string = final_normalize(string) string = final_normalize(string)
...@@ -207,7 +209,7 @@ class WikiQuestionGenerator(object): ...@@ -207,7 +209,7 @@ class WikiQuestionGenerator(object):
self.dev_loader = WikiQuestionLoader(dev_name, root_folder) self.dev_loader = WikiQuestionLoader(dev_name, root_folder)
self.test_loader = WikiQuestionLoader(test_name, root_folder) self.test_loader = WikiQuestionLoader(test_name, root_folder)
self.bad_examples = 0 self.bad_examples = 0
self.root_folder = root_folder self.root_folder = root_folder
self.data_folder = os.path.join(self.root_folder, "annotated/data") self.data_folder = os.path.join(self.root_folder, "annotated/data")
self.annotated_examples = {} self.annotated_examples = {}
self.annotated_tables = {} self.annotated_tables = {}
...@@ -298,7 +300,7 @@ class WikiQuestionGenerator(object): ...@@ -298,7 +300,7 @@ class WikiQuestionGenerator(object):
question_id, question, target_canon, context) question_id, question, target_canon, context)
self.annotated_tables[context] = [] self.annotated_tables[context] = []
counter += 1 counter += 1
print "Annotated examples loaded ", len(self.annotated_examples) print("Annotated examples loaded ", len(self.annotated_examples))
f.close() f.close()
def is_number_column(self, a): def is_number_column(self, a):
......
research/next_frame_prediction/cross_conv/eval.py
View file @ d5e826e3
...@@ -20,6 +20,7 @@ import sys ...@@ -20,6 +20,7 @@ import sys
import time import time
import numpy as np import numpy as np
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
import model as cross_conv_model import model as cross_conv_model
......
research/next_frame_prediction/cross_conv/example_gen.py
View file @ d5e826e3
...@@ -18,6 +18,7 @@ import random ...@@ -18,6 +18,7 @@ import random
import sys import sys
import numpy as np import numpy as np
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
......
research/next_frame_prediction/cross_conv/model.py
View file @ d5e826e3
...@@ -20,6 +20,7 @@ https://arxiv.org/pdf/1607.02586v1.pdf ...@@ -20,6 +20,7 @@ https://arxiv.org/pdf/1607.02586v1.pdf
import math import math
import sys import sys
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
slim = tf.contrib.slim slim = tf.contrib.slim
......
research/next_frame_prediction/cross_conv/reader.py
View file @ d5e826e3
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
"""Read image sequence.""" """Read image sequence."""
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
......
research/next_frame_prediction/cross_conv/sprites_gen.py
View file @ d5e826e3
...@@ -21,6 +21,7 @@ import sys ...@@ -21,6 +21,7 @@ import sys
import numpy as np import numpy as np
import scipy.misc import scipy.misc
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
......
research/object_detection/dataset_tools/create_oid_tf_record.py
View file @ d5e826e3
...@@ -96,7 +96,7 @@ def main(_): ...@@ -96,7 +96,7 @@ def main(_):
tf_example = oid_tfrecord_creation.tf_example_from_annotations_data_frame( tf_example = oid_tfrecord_creation.tf_example_from_annotations_data_frame(
image_annotations, label_map, encoded_image) image_annotations, label_map, encoded_image)
if tf_example: if tf_example:
shard_idx = long(image_id, 16) % FLAGS.num_shards shard_idx = int(image_id, 16) % FLAGS.num_shards
output_tfrecords[shard_idx].write(tf_example.SerializeToString()) output_tfrecords[shard_idx].write(tf_example.SerializeToString())
......
research/object_detection/dataset_tools/oid_tfrecord_creation.py
View file @ d5e826e3
...@@ -18,6 +18,7 @@ from __future__ import absolute_import ...@@ -18,6 +18,7 @@ from __future__ import absolute_import
from __future__ import division from __future__ import division
from __future__ import print_function from __future__ import print_function
from six.moves import xrange
import tensorflow as tf import tensorflow as tf
from object_detection.core import standard_fields from object_detection.core import standard_fields
......
research/object_detection/utils/np_box_list_ops.py
View file @ d5e826e3
...@@ -21,6 +21,7 @@ Example box operations that are supported: ...@@ -21,6 +21,7 @@ Example box operations that are supported:
""" """
import numpy as np import numpy as np
from six.moves import xrange
from object_detection.utils import np_box_list from object_detection.utils import np_box_list
from object_detection.utils import np_box_ops from object_detection.utils import np_box_ops
......
research/object_detection/utils/ops.py
View file @ d5e826e3
...@@ -203,9 +203,9 @@ def padded_one_hot_encoding(indices, depth, left_pad): ...@@ -203,9 +203,9 @@ def padded_one_hot_encoding(indices, depth, left_pad):
TODO: add runtime checks for depth and indices. TODO: add runtime checks for depth and indices.
""" """
if depth < 0 or not isinstance(depth, (int, long) if six.PY2 else int): if depth < 0 or not isinstance(depth, six.integer_types):
raise ValueError('`depth` must be a non-negative integer.') raise ValueError('`depth` must be a non-negative integer.')
if left_pad < 0 or not isinstance(left_pad, (int, long) if six.PY2 else int): if left_pad < 0 or not isinstance(left_pad, six.integer_types):
raise ValueError('`left_pad` must be a non-negative integer.') raise ValueError('`left_pad` must be a non-negative integer.')
if depth == 0: if depth == 0:
return None return None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment