Better error message for improperly formatted dictionaries

Summary: Pull Request resolved: https://github.com/pytorch/fairseq/pull/468

Differential Revision: D13802590

Pulled By: myleott

fbshipit-source-id: e374e38e74dc91bda0579ae41e26289fb0ba56a2

fairseq/data/dictionary.py

@@ -172,13 +172,15 @@ class Dictionary(object):
                         return cls.load(fd)
             except FileNotFoundError as fnfe:
                 raise fnfe
-            except Exception:
+            except UnicodeError:
                 raise Exception("Incorrect encoding detected in {}, please "
                                 "rebuild the dataset".format(f))
 
         d = cls()
         for line in f.readlines():
             idx = line.rfind(' ')
+            if idx == -1:
+                raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'")
             word = line[:idx]
             count = int(line[idx+1:])
             d.indices[word] = len(d.symbols)