"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "6f877d9daf36788bad4fd228930939fed6ab12bd"
Unverified Commit e8db8b84 authored by Sylvain Gugger's avatar Sylvain Gugger Committed by GitHub
Browse files

Remove unused arguments in Multiple Choice example (#4853)

* Remove unused arguments

* Formatting

* Remove second todo comment
parent 29c36e9f
...@@ -121,16 +121,7 @@ if is_torch_available(): ...@@ -121,16 +121,7 @@ if is_torch_available():
else: else:
examples = processor.get_train_examples(data_dir) examples = processor.get_train_examples(data_dir)
logger.info("Training examples: %s", len(examples)) logger.info("Training examples: %s", len(examples))
# TODO clean up all this to leverage built-in features of tokenizers self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
self.features = convert_examples_to_features(
examples,
label_list,
max_seq_length,
tokenizer,
pad_on_left=bool(tokenizer.padding_side == "left"),
pad_token=tokenizer.pad_token_id,
pad_token_segment_id=tokenizer.pad_token_type_id,
)
logger.info("Saving features into cached file %s", cached_features_file) logger.info("Saving features into cached file %s", cached_features_file)
torch.save(self.features, cached_features_file) torch.save(self.features, cached_features_file)
...@@ -172,16 +163,8 @@ if is_tf_available(): ...@@ -172,16 +163,8 @@ if is_tf_available():
else: else:
examples = processor.get_train_examples(data_dir) examples = processor.get_train_examples(data_dir)
logger.info("Training examples: %s", len(examples)) logger.info("Training examples: %s", len(examples))
# TODO clean up all this to leverage built-in features of tokenizers
self.features = convert_examples_to_features( self.features = convert_examples_to_features(examples, label_list, max_seq_length, tokenizer,)
examples,
label_list,
max_seq_length,
tokenizer,
pad_on_left=bool(tokenizer.padding_side == "left"),
pad_token=tokenizer.pad_token_id,
pad_token_segment_id=tokenizer.pad_token_type_id,
)
def gen(): def gen():
for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"): for (ex_index, ex) in tqdm.tqdm(enumerate(self.features), desc="convert examples to features"):
...@@ -506,14 +489,7 @@ class ArcProcessor(DataProcessor): ...@@ -506,14 +489,7 @@ class ArcProcessor(DataProcessor):
def convert_examples_to_features( def convert_examples_to_features(
examples: List[InputExample], examples: List[InputExample], label_list: List[str], max_length: int, tokenizer: PreTrainedTokenizer,
label_list: List[str],
max_length: int,
tokenizer: PreTrainedTokenizer,
pad_token_segment_id=0,
pad_on_left=False,
pad_token=0,
mask_padding_with_zero=True,
) -> List[InputFeatures]: ) -> List[InputFeatures]:
""" """
Loads a data file into a list of `InputFeatures` Loads a data file into a list of `InputFeatures`
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment