Merge pull request #1233 from searchivarius/master

Fix to prevent crashing on assert len(tokens_b)>=1

examples/lm_finetuning/pregenerate_training_data.py

@@ -329,6 +329,7 @@ def main():
                     doc = []
                 else:
                     tokens = tokenizer.tokenize(line)
+                    if tokens:
                         doc.append(tokens)
             if doc:
                 docs.add_document(doc)  # If the last doc didn't end on a newline, make sure it still gets added