Skip to content

GitLab

Commit e33fb967 authored by Chen Chen's avatar Chen Chen Committed by A. Unique TensorFlower
Browse files

Internal Change 

PiperOrigin-RevId: 295222307
parent 5f296bbe
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 0 deletions
official/nlp/bert/input_pipeline.py
View file @ e33fb967
...@@ -78,6 +78,10 @@ def create_pretrain_dataset(input_patterns, ...@@ -78,6 +78,10 @@ def create_pretrain_dataset(input_patterns,
tf.io.FixedLenFeature([1], tf.int64), tf.io.FixedLenFeature([1], tf.int64),
} }
for input_pattern in input_patterns:
if not tf.io.gfile.glob(input_pattern):
raise ValueError('%s does not match any files.' % input_pattern)
dataset = tf.data.Dataset.list_files(input_patterns, shuffle=is_training) dataset = tf.data.Dataset.list_files(input_patterns, shuffle=is_training)
if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1: if input_pipeline_context and input_pipeline_context.num_input_pipelines > 1:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment