More fixes

README.md

@@ -151,8 +151,8 @@ BPE continuation markers can be removed with the `--remove-bpe` flag.
 
 We provide the following pre-trained fully convolutional sequence-to-sequence models:
 
-* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/faiseq-py/models/wmt14.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies
-* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/faiseq-py/models/wmt14.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies
+* [wmt14.en-fr.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-French](http://statmt.org/wmt14/translation-task.html#Download) including vocabularies
+* [wmt14.en-de.fconv-py.tar.bz2](https://s3.amazonaws.com/fairseq-py/models/wmt14.en-de.fconv-py.tar.bz2): Pre-trained model for [WMT14 English-German](https://nlp.stanford.edu/projects/nmt) including vocabularies
 
 In addition, we provide pre-processed and binarized test sets for the models above:
 * [wmt14.en-fr.newstest2014.tar.bz2](https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2): newstest2014 test set for WMT14 English-French
@@ -161,23 +161,20 @@ In addition, we provide pre-processed and binarized test sets for the models abo
 
 Generation with the binarized test sets can be run in batch mode as follows, e.g. for English-French on a GTX-1080ti:
 ```
-$ curl https://s3.amazonaws.com/faiseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin
+$ curl https://s3.amazonaws.com/fairseq-py/models/wmt14.en-fr.fconv-py.tar.bz2 | tar xvjf - -C data-bin
 $ curl https://s3.amazonaws.com/fairseq-py/data/wmt14.en-fr.newstest2014.tar.bz2 | tar xvjf - -C data-bin
 $ python generate.py data-bin/wmt14.en-fr.newstest2014  \
   --path data-bin/wmt14.en-fr.fconv-py/model.pt \
   --beam 5 --batch-size 128 --remove-bpe | tee /tmp/gen.out
 ...
-| Translated 3003 sentences (95451 tokens) in 136.3s (700.49 tokens/s)
-| Timings: setup 0.1s (0.1%), encoder 1.9s (1.4%), decoder 108.9s (79.9%), search_results 0.0s (0.0%), search_prune 12.5s (9.2%)
-TODO: update scores (should be same as score.py)
-| BLEU4 = 43.43, 68.2/49.2/37.4/28.8 (BP=0.996, ratio=1.004, sys_len=92087, ref_len=92448)
+| Translated 3003 sentences (95451 tokens) in 81.3s (1174.33 tokens/s)
+| Generate test with beam=5: BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)
 
 # Scoring with score.py:
-$ grep ^H /tmp/gen.out | cut -f3- | sed 's/@@ //g' > /tmp/gen.out.sys
-$ grep ^T /tmp/gen.out | cut -f2- | sed 's/@@ //g' > /tmp/gen.out.ref
+$ grep ^H /tmp/gen.out | cut -f3- > /tmp/gen.out.sys
+$ grep ^T /tmp/gen.out | cut -f2- > /tmp/gen.out.ref
 $ python score.py --sys /tmp/gen.out.sys --ref /tmp/gen.out.ref
-TODO: update scores
-BLEU4 = 40.55, 67.6/46.5/34.0/25.3 (BP=1.000, ratio=0.998, sys_len=81369, ref_len=81194)
+BLEU4 = 40.23, 67.5/46.4/33.8/25.0 (BP=0.997, ratio=1.003, syslen=80963, reflen=81194)
 ```
 
 # Join the fairseq community

fairseq/utils.py

@@ -6,8 +6,11 @@
 # can be found in the PATENTS file in the same directory.
 #
 
+import logging
 import os
 import torch
+import traceback
+
 from torch.autograd import Variable
 from torch.serialization import default_restore_location
 
@@ -39,8 +42,8 @@ def torch_persistent_save(*args, **kwargs):
         try:
             return torch.save(*args, **kwargs)
         except:
-            if i == 3:
-                raise
+            if i == 2:
+                logging.error(traceback.format_exc())
 
 
 def save_checkpoint(args, epoch, batch_offset, model, optimizer, lr_scheduler, val_loss=None):

generate.py

@@ -110,25 +110,16 @@ def main():
             display_hypotheses(None, tokens, line, None, hypos[:min(len(hypos), args.nbest)])
 
     else:
-        non_bpe_dict = {}
-        def maybe_remove_bpe_and_reindex(tokens):
-            """Helper for removing BPE symbols from a tensor of indices.
-
-            If BPE removal is enabled, the returned tensor is reindexed
-            using a new dictionary that is created on-the-fly."""
+        def maybe_remove_bpe(tokens):
+            """Helper for removing BPE symbols from a hypothesis."""
             if not args.remove_bpe:
                 return tokens
             assert (tokens == dataset.dst_dict.pad()).sum() == 0
-            return torch.IntTensor([
-                non_bpe_dict.setdefault(w, len(non_bpe_dict))
-                for w in to_sentence(dataset.dst_dict, tokens, bpe_symbol).split(' ')
-            ])
+            hypo_minus_bpe = to_sentence(dataset.dst_dict, tokens, bpe_symbol)
+            return tokenizer.Tokenizer.tokenize(hypo_minus_bpe, dataset.dst_dict, add_if_not_exist=True)
 
         # Generate and compute BLEU score
-        scorer = bleu.Scorer(
-            dataset.dst_dict.pad() if not args.remove_bpe else -1,
-            dataset.dst_dict.eos() if not args.remove_bpe else -1,
-            dataset.dst_dict.unk())
+        scorer = bleu.Scorer(dataset.dst_dict.pad(), dataset.dst_dict.eos(), dataset.dst_dict.unk())
         itr = dataset.dataloader(args.gen_subset, batch_size=args.batch_size, max_positions=args.max_positions)
         num_sentences = 0
         with progress_bar(itr, smoothing=0, leave=False) as t:
@@ -140,7 +131,7 @@ def main():
             for id, src, ref, hypos in translations:
                 ref = ref.int().cpu()
                 top_hypo = hypos[0]['tokens'].int().cpu()
-                scorer.add(maybe_remove_bpe_and_reindex(ref), maybe_remove_bpe_and_reindex(top_hypo))
+                scorer.add(maybe_remove_bpe(ref), maybe_remove_bpe(top_hypo))
                 display_hypotheses(id, src, None, ref, hypos[:min(len(hypos), args.nbest)])
 
                 wps_meter.update(src.size(0))

score.py

@@ -16,7 +16,7 @@ from fairseq import bleu, dictionary, tokenizer
 def main():
     parser = argparse.ArgumentParser(description='Command-line script for BLEU scoring.')
     parser.add_argument('-s', '--sys', default='-', help='system output')
-    parser.add_argument('-r', '--ref', default='', help='references')
+    parser.add_argument('-r', '--ref', required=True, help='references')
     parser.add_argument('-o', '--order', default=4, metavar='N',
                         type=int, help='consider ngrams up to this order')
     parser.add_argument('--ignore-case', action='store_true',