Skip to content

GitLab

Commit 079bfb32 authored by altsoph's avatar altsoph Committed by Lysandre Debut
Browse files

Evaluation fixed.

parent 438f2730
Show whitespace changes
Inline Side-by-side
Showing with 1 addition and 2 deletions
examples/run_lm_finetuning.py
View file @ 079bfb32
...@@ -86,7 +86,6 @@ class TextDataset(Dataset): ...@@ -86,7 +86,6 @@ class TextDataset(Dataset):
# Note that we are loosing the last truncated example here for the sake of simplicity (no padding) # Note that we are loosing the last truncated example here for the sake of simplicity (no padding)
# If your dataset is small, first you should loook for a bigger one :-) and second you # If your dataset is small, first you should loook for a bigger one :-) and second you
# can change this behavior by adding (model specific) padding. # can change this behavior by adding (model specific) padding.
self.examples.append(tokenizer.build_inputs_with_special_tokens(tokenized_text[-block_size:])) # DIRTY!
logger.info("Saving features into cached file %s", cached_features_file) logger.info("Saving features into cached file %s", cached_features_file)
with open(cached_features_file, 'wb') as handle: with open(cached_features_file, 'wb') as handle:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment