thepile.yaml

_target_: src.datamodules.language_modeling_hf.LMDataModule
dataset_name: the_pile
dataset_config_name: null
tokenizer_name: gpt2
cache_dir: ${oc.env:DATA_DIR,${data_dir}}/the_pile/cache
max_length: 2048
add_eos: True
batch_size: 4  # per GPU
batch_size_eval: ${eval:${.batch_size} * 2}
num_workers: 64  # For preprocessing only
use_shmem: False
shuffle: True
pin_memory: True
__train_len: ${div_up:374337375694, ${.max_length}}