Unverified Commit 033c3ed9 authored by Stas Bekman's avatar Stas Bekman Committed by GitHub
Browse files

[examples/summarization] deal with None in data records (#14816)

* [examples/summarization] deal with None in data records

* rewrite to use a simpler (slower) variant
parent c075fb78
......@@ -436,8 +436,14 @@ def main():
)
def preprocess_function(examples):
inputs = examples[text_column]
targets = examples[summary_column]
# remove pairs where at least one record is None
inputs, targets = [], []
for i in range(len(examples[text_column])):
if examples[text_column][i] is not None and examples[summary_column][i] is not None:
inputs.append(examples[text_column][i])
targets.append(examples[summary_column][i])
inputs = [prefix + inp for inp in inputs]
model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, padding=padding, truncation=True)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment