Unverified Commit cdf4cd70 authored by Tomo Lazovich's avatar Tomo Lazovich Committed by GitHub
Browse files

[squad] add version tag to squad cache (#5669)

parent 223084e4
...@@ -113,9 +113,12 @@ class SquadDataset(Dataset): ...@@ -113,9 +113,12 @@ class SquadDataset(Dataset):
raise KeyError("mode is not a valid split name") raise KeyError("mode is not a valid split name")
self.mode = mode self.mode = mode
# Load data features from cache or dataset file # Load data features from cache or dataset file
version_tag = "v2" if args.version_2_with_negative else "v1"
cached_features_file = os.path.join( cached_features_file = os.path.join(
cache_dir if cache_dir is not None else args.data_dir, cache_dir if cache_dir is not None else args.data_dir,
"cached_{}_{}_{}".format(mode.value, tokenizer.__class__.__name__, str(args.max_seq_length),), "cached_{}_{}_{}_{}".format(
mode.value, tokenizer.__class__.__name__, str(args.max_seq_length), version_tag,
),
) )
# Make sure only the first process in distributed training processes the dataset, # Make sure only the first process in distributed training processes the dataset,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment