Commit c2793c1b authored by Frederick Liu's avatar Frederick Liu Committed by A. Unique TensorFlower
Browse files

[core] Always call build_and_prepare since it is no op if exist.

https://www.tensorflow.org/datasets/api_docs/python/tfds/load

PiperOrigin-RevId: 360237231
parent 3dac6569
...@@ -63,7 +63,6 @@ class DataConfig(base_config.Config): ...@@ -63,7 +63,6 @@ class DataConfig(base_config.Config):
The default behavior is that the dataset creates anonymous, exclusively The default behavior is that the dataset creates anonymous, exclusively
owned jobs. owned jobs.
tfds_data_dir: A str specifying the directory to read/write TFDS data. tfds_data_dir: A str specifying the directory to read/write TFDS data.
tfds_download: A bool to indicate whether to download data using TFDS.
tfds_as_supervised: A bool. When loading dataset from TFDS, if True, the tfds_as_supervised: A bool. When loading dataset from TFDS, if True, the
returned tf.data.Dataset will have a 2-tuple structure (input, label) returned tf.data.Dataset will have a 2-tuple structure (input, label)
according to builder.info.supervised_keys; if False, the default, the according to builder.info.supervised_keys; if False, the default, the
...@@ -89,7 +88,6 @@ class DataConfig(base_config.Config): ...@@ -89,7 +88,6 @@ class DataConfig(base_config.Config):
tf_data_service_address: Optional[str] = None tf_data_service_address: Optional[str] = None
tf_data_service_job_name: Optional[str] = None tf_data_service_job_name: Optional[str] = None
tfds_data_dir: str = "" tfds_data_dir: str = ""
tfds_download: bool = False
tfds_as_supervised: bool = False tfds_as_supervised: bool = False
tfds_skip_decoding_feature: str = "" tfds_skip_decoding_feature: str = ""
......
...@@ -101,7 +101,6 @@ class InputReader: ...@@ -101,7 +101,6 @@ class InputReader:
self._deterministic = params.deterministic self._deterministic = params.deterministic
self._sharding = params.sharding self._sharding = params.sharding
self._tfds_split = params.tfds_split self._tfds_split = params.tfds_split
self._tfds_download = params.tfds_download
self._tfds_as_supervised = params.tfds_as_supervised self._tfds_as_supervised = params.tfds_as_supervised
self._tfds_skip_decoding_feature = params.tfds_skip_decoding_feature self._tfds_skip_decoding_feature = params.tfds_skip_decoding_feature
...@@ -232,8 +231,8 @@ class InputReader: ...@@ -232,8 +231,8 @@ class InputReader:
input_context: Optional[tf.distribute.InputContext] = None input_context: Optional[tf.distribute.InputContext] = None
) -> tf.data.Dataset: ) -> tf.data.Dataset:
"""Reads a dataset from tfds.""" """Reads a dataset from tfds."""
if self._tfds_download: # No op if exist.
self._tfds_builder.download_and_prepare() self._tfds_builder.download_and_prepare()
read_config = tfds.ReadConfig( read_config = tfds.ReadConfig(
interleave_cycle_length=self._cycle_length, interleave_cycle_length=self._cycle_length,
......
...@@ -164,7 +164,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase, ...@@ -164,7 +164,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase,
input_path='' if use_tfds else tf_record_path, input_path='' if use_tfds else tf_record_path,
tfds_name='glue/mrpc' if use_tfds else '', tfds_name='glue/mrpc' if use_tfds else '',
tfds_split='train' if use_tfds else '', tfds_split='train' if use_tfds else '',
tfds_download=True,
text_fields=text_fields, text_fields=text_fields,
global_batch_size=batch_size, global_batch_size=batch_size,
seq_length=seq_length, seq_length=seq_length,
...@@ -196,7 +195,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase, ...@@ -196,7 +195,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase,
input_path='' if use_tfds else tf_record_path, input_path='' if use_tfds else tf_record_path,
tfds_name='glue/mrpc' if use_tfds else '', tfds_name='glue/mrpc' if use_tfds else '',
tfds_split='train' if use_tfds else '', tfds_split='train' if use_tfds else '',
tfds_download=True,
text_fields=text_fields, text_fields=text_fields,
global_batch_size=batch_size, global_batch_size=batch_size,
seq_length=seq_length, seq_length=seq_length,
...@@ -230,7 +228,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase, ...@@ -230,7 +228,6 @@ class SentencePredictionTfdsDataLoaderTest(tf.test.TestCase,
input_path='' if use_tfds else tf_record_path, input_path='' if use_tfds else tf_record_path,
tfds_name='glue/mrpc' if use_tfds else '', tfds_name='glue/mrpc' if use_tfds else '',
tfds_split='train' if use_tfds else '', tfds_split='train' if use_tfds else '',
tfds_download=True,
text_fields=text_fields, text_fields=text_fields,
global_batch_size=batch_size, global_batch_size=batch_size,
seq_length=seq_length, seq_length=seq_length,
......
...@@ -20,7 +20,6 @@ task: ...@@ -20,7 +20,6 @@ task:
tfds_name: 'imagenet2012' tfds_name: 'imagenet2012'
tfds_split: 'train' tfds_split: 'train'
tfds_data_dir: '~/tensorflow_datasets/' tfds_data_dir: '~/tensorflow_datasets/'
tfds_download: true
is_training: true is_training: true
global_batch_size: 16 # default = 128 global_batch_size: 16 # default = 128
dtype: 'float16' dtype: 'float16'
...@@ -29,7 +28,6 @@ task: ...@@ -29,7 +28,6 @@ task:
tfds_name: 'imagenet2012' tfds_name: 'imagenet2012'
tfds_split: 'validation' tfds_split: 'validation'
tfds_data_dir: '~/tensorflow_datasets/' tfds_data_dir: '~/tensorflow_datasets/'
tfds_download: true
is_training: true is_training: true
global_batch_size: 16 # default = 128 global_batch_size: 16 # default = 128
dtype: 'float16' dtype: 'float16'
......
...@@ -20,7 +20,6 @@ task: ...@@ -20,7 +20,6 @@ task:
tfds_name: 'imagenet2012' tfds_name: 'imagenet2012'
tfds_split: 'train' tfds_split: 'train'
tfds_data_dir: '~/tensorflow_datasets/' tfds_data_dir: '~/tensorflow_datasets/'
tfds_download: true
is_training: true is_training: true
global_batch_size: 16 # default = 128 global_batch_size: 16 # default = 128
dtype: 'float16' dtype: 'float16'
...@@ -29,7 +28,6 @@ task: ...@@ -29,7 +28,6 @@ task:
tfds_name: 'imagenet2012' tfds_name: 'imagenet2012'
tfds_split: 'validation' tfds_split: 'validation'
tfds_data_dir: '~/tensorflow_datasets/' tfds_data_dir: '~/tensorflow_datasets/'
tfds_download: true
is_training: true is_training: true
global_batch_size: 16 # default = 128 global_batch_size: 16 # default = 128
dtype: 'float16' dtype: 'float16'
......
...@@ -52,7 +52,6 @@ class DataConfig(cfg.DataConfig): ...@@ -52,7 +52,6 @@ class DataConfig(cfg.DataConfig):
decoder = None decoder = None
parser: Parser = Parser() parser: Parser = Parser()
shuffle_buffer_size: int = 10 shuffle_buffer_size: int = 10
tfds_download: bool = False
class YoloDetectionInputTest(tf.test.TestCase, parameterized.TestCase): class YoloDetectionInputTest(tf.test.TestCase, parameterized.TestCase):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment