gpt2

CONTRIBUTING.md

+## Contributing to LiBai
\ No newline at end of file

LICENSE

+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

+# Generative Pre-Training2(GPT2)
+## 模型介绍
+GPT2模型：第二代生成式预训练模型（Generative Pre-Training2）。
+## 模型结构
+GPT2使用 Transformer 的 Decoder 结构，并对 Transformer Decoder 进行了一些改动，原本的 Decoder 包含了两个 Multi-Head Attention 结构，GPT2 只保留了 Mask Multi-Head Attention。
+
+我们为了用户可以使用OneFlow-Libai快速验证GPT2模型预训练，统计性能或验证精度，提供了一个GPT2网络示例，主要网络参数：
+
+```
+model.cfg.num_attention_heads = 16
+model.cfg.hidden_size = 384
+model.cfg.ffn_hidden_size = 1536
+model.cfg.hidden_layers = 6
+model.cfg.max_seq_length = 1024
+```
+
+完整的GPT2网络配置在configs/common/model/gpt.py中
+
+同时，我们提供了一个更大的GPT2-13B网络为了用户可以快速在DCU集群上使用OneFlow-Libai进行较大规模的混合并行分布式预训练验证（该网络可能并不具有实际训练价值），该网络结构在GPT2基础上进行扩充，主要网络参数如下，参数量共有13.1B：
+
+```
+model.cfg.num_attention_heads = 32
+model.cfg.hidden_size = 4096
+model.cfg.ffn_hidden_size = 4096*4
+model.cfg.hidden_layers = 64
+model.cfg.max_seq_length = 1024
+```
+
+## 数据集
+我们在libai目录下集成了部分小数据集供用户快速验证：
+
+    ./nlp_data
+## GPT2预训练
+
+### 环境配置
+
+推荐使用docker方式运行，提供[光源](https://www.sourcefind.cn/#/service-details)拉取的docker镜像：image.sourcefind.cn:5000/dcu/admin/base/oneflow:0.9.1-centos7.6-dtk-22.10.1-py39-latest
+
+进入docker：
+
+    cd libai
+    pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install pybind11 -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install -e . -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install oneflow-0.9.1+dtk2210.git.8ea46d6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+
+### 训练
+
+该预训练脚本运行环境为1节点，4张DCU-Z100-16G。
+
+并行配置策略在configs/gpt2_pretrain.py中，使用自动混合精度：
+
+```
+train.amp.enabled = True
+train.train_micro_batch_size = 4
+train.dist.data_parallel_size = 4
+train.dist.tensor_parallel_size = 1
+train.dist.pipeline_parallel_size = 1
+```
+
+预训练命令：
+
+    cd libai
+    bash tools/train.sh tools/train_net.py configs/gpt2_pretrain.py 4
+
+### 性能和收敛性
+
+训练数据：[https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset](链接)
+
+使用的GPGPU：4张DCU-Z100-16G。
+
+模型性能及收敛性：
+
+| 卡数 | 分布式工具 |       性能       |            收敛性             |
+| :--: | :--------: | :--------------: | :---------------------------: |
+|  4   | Libai-main | 129.55 samples/s | total_loss: 4.336/10000 iters |
+
+## GPT2-13B预训练
+
+### 环境配置
+
+要求DCU集群Slurm环境正常。
+
+推荐用户使用预编译好的python3.9包来快速建立python3虚拟环境：
+
+    cd libai
+    export PYTHON3_LIB_PATH=/python_lib_path
+    virtualenv -p /python_bin_path/python3 --system-site-packages venv_oneflow
+    source env.sh	#进入venv_oneflow虚拟环境
+    
+    pip3 install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple	#更新pip
+    pip3 install -r requirements.txt -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install pybind11 -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install -e . -i https://mirrors.aliyun.com/pypi/simple
+    pip3 install oneflow-0.9.1+dtk2210.git.8ea46d6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
+
+### 训练
+该预训练脚本需要24个节点，每节点4张DCU-Z100-16G。
+
+混合并行配置策略在configs/gpt2-13B_pretrain.py中，使用自动混合精度：
+
+```
+train.amp.enabled = True
+train.train_micro_batch_size = 2
+train.num_accumulation_steps = 4
+train.activation_checkpoint.enabled = True
+train.zero_optimization.enabled = True
+train.zero_optimization.stage = 1
+train.dist.data_parallel_size = 6
+train.dist.tensor_parallel_size = 4
+train.dist.pipeline_parallel_size = 4 
+```
+
+进入登陆节点，预训练命令：
+
+    cd libai
+    source submit_job.sh
+    tail -f log/xxx.out.log	#查看输出log
+    tail -f log/xxx.err.log	#查看错误log
+
+### 性能和收敛性
+
+训练数据：[https://oneflow-static.oss-cn-beijing.aliyuncs.com/ci-files/dataset/libai/gpt_dataset](链接)
+
+使用的GPGPU：96张DCU-Z100-16G。
+
+模型性能及收敛性：
+| 卡数 | 分布式工具 | 性能 | 收敛性 |
+| :------: | :------: | :------: |:------: |
+| 96 | Libai-main | 2.27 samples/s | total_loss: 5.56/1299 iters |
+## 参考
+* https://libai.readthedocs.io/en/latest/tutorials/get_started/quick_run.html
+* https://github.com/Oneflow-Inc/oneflow
+* https://github.com/Oneflow-Inc/libai/blob/main/docs/source/notes/FAQ.md
\ No newline at end of file

changelog.md

+## Changelog
+
+### Beta 0.1.0 (22/03/2022)
+
+**New Features:**
+- Support Data Parallelism
+- Support 1D Tensor Parallelism
+- Support Pipeline Parallelism
+- Unified distributed Layers for both single-GPU and multi-GPU training
+- `LazyConfig` system for more flexible syntax and no predefined structures
+- Easy-to-use trainer and engine
+- Support both CV and NLP data processing
+- Mixed Precision Training
+- Activation Checkpointing
+- Gradient Accumulation
+- Gradient Clipping
+- Zero Redundancy Optimizer (ZeRO)
+
+**Supported Models:**
+- Support 3D parallel [BERT](https://arxiv.org/abs/1810.04805) model
+- Support 3D parallel [GPT-2](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) model
+- Support 3D parallel [T5](https://arxiv.org/abs/1910.10683) model
+- Support 3D parallel [Vision Transformer](https://arxiv.org/abs/2010.11929)
+- Support Data parallel [Swin Transformer](https://arxiv.org/abs/2103.14030) model
+- Support finetune task in [QQP project](/projects/QQP/)
+- Support text classification task in [text classification project](/projects/text_classification/)

configs/bert_classification.py

+from libai.config import LazyCall
+from libai.models.bert_model import BertForClassification
+from .common.models.bert import cfg as bert_cfg
+from .common.models.graph import graph
+from .common.train import train
+from .common.optim import optim
+from .common.data.bert_dataset import tokenization, dataloader
+
+vocab_file = "./data_test/bert_data/bert-base-chinese-vocab.txt"
+data_prefix = "./data_test/bert_data/loss_compara_content_sentence"
+
+dataloader.train.dataset[0].data_prefix = data_prefix
+dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix
+
+bert_cfg.num_labels = 2
+bert_cfg.classifier_dropout = 0.1
+
+model = LazyCall(BertForClassification)(cfg=bert_cfg)
+tokenization.tokenizer.vocab_file = vocab_file
+
+model.cfg.vocab_size = 21128
+model.cfg.intermediate_size = 3072
+model.cfg.num_attention_heads = 12
+model.cfg.hidden_layers = 12
+model.cfg.hidden_size = 768
+
+train.amp.enabled = True
+train.activation_checkpoint.enabled = True
+train.dist.pipeline_num_layers = model.cfg.hidden_layers
+
+train.output_dir = "output/bert_classification_output"

configs/bert_large_pretrain.py

+from libai.config import LazyCall
+from libai.evaluation import PPLEvaluator
+from .common.models.bert import pretrain_model as model
+from .common.models.graph import graph
+from .common.train import train
+from .common.optim import optim
+from .common.data.bert_dataset import dataloader, tokenization
+
+vocab_file = "./nlp_data/bert-base-chinese-vocab.txt"
+data_prefix = "./nlp_data/data/loss_compara_content_sentence"
+
+tokenization.tokenizer.vocab_file = vocab_file
+dataloader.train.dataset[0].data_prefix = data_prefix
+dataloader.train.dataset[0].indexed_dataset.data_prefix = data_prefix
+
+# Bert-large model config
+model.cfg.num_attention_heads = 40
+model.cfg.hidden_size = 5120
+model.cfg.intermediate_size = 5120*4
+model.cfg.hidden_layers = 20
+
+train.input_placement_device = "cpu"
+train.dist.pipeline_num_layers = model.cfg.hidden_layers
+train.amp.enabled = True
+graph.auto_parallel.enabled = False
+train.train_micro_batch_size = 8
+
+train.num_accumulation_steps = 4
+train.activation_checkpoint.enabled = True
+train.zero_optimization.enabled = True
+train.zero_optimization.stage = 1
+
+train.dist.data_parallel_size = 12
+train.dist.tensor_parallel_size = 4
+train.dist.pipeline_parallel_size = 2
+
+train.dist.custom_pipeline_stage_id = [0]*8 + [1]*12
+
+for ds in dataloader.train.dataset:
+    ds.max_seq_length = model.cfg.max_position_embeddings
+
+train.evaluation.evaluator = LazyCall(PPLEvaluator)()
+
+train.output_dir = "output/bert_output"

configs/common/data/bert_dataset.py

+from libai.config import LazyCall
+from omegaconf import OmegaConf
+from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader
+from libai.data.datasets import BertDataset
+from libai.data.data_utils import get_indexed_dataset
+
+from libai.tokenizer import BertTokenizer
+
+
+tokenization = OmegaConf.create()
+
+tokenization.tokenizer = LazyCall(BertTokenizer)(
+    vocab_file="bert-base-chinese-vocab.txt",
+    do_lower_case=True,
+    do_chinese_wwm=True,
+)
+tokenization.append_eod = False
+tokenization.make_vocab_size_divisible_by = 128
+
+dataloader = OmegaConf.create()
+
+dataloader.train = LazyCall(build_nlp_train_val_test_loader)(
+    dataset=[
+        LazyCall(BertDataset)(
+            name="bert",
+            data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+            indexed_dataset=LazyCall(get_indexed_dataset)(
+                data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+                data_impl="mmap",
+                skip_warmup=False,
+            ),
+            max_seq_length=512,
+            mask_lm_prob=0.15,
+            short_seq_prob=0.1,
+            binary_head=True,
+            seed=1234,
+            masking_style="bert-cn-wwm",
+        ),
+    ],
+    train_val_test_num_samples=None,  # a hint for deferred assignment
+    splits=[[949.0, 50.0, 1.0]],
+    weights=[1.0],
+    num_workers=4,
+)

configs/common/data/cifar100.py

+from omegaconf import OmegaConf
+from flowvision import transforms
+from flowvision.data.mixup import Mixup
+from flowvision.transforms import InterpolationMode
+from flowvision.transforms.functional import str_to_interp_mode
+
+from libai.data.datasets import CIFAR100Dataset
+from libai.data.build import build_image_train_loader, build_image_test_loader
+from libai.config import LazyCall
+
+# mean and std of cifar100 dataset
+CIFAR100_TRAIN_MEAN = (0.5070751592371323, 0.48654887331495095, 0.4409178433670343)
+CIFAR100_TRAIN_STD = (0.2673342858792401, 0.2564384629170883, 0.27615047132568404)
+
+train_aug = LazyCall(transforms.Compose)(
+    transforms=[
+        LazyCall(transforms.RandomResizedCrop)(
+            size=(224, 224),
+            scale=(0.08, 1.0),
+            ratio=(3.0 / 4.0, 4.0 / 3.0),
+            interpolation=str_to_interp_mode("bicubic"),
+        ),
+        LazyCall(transforms.RandomHorizontalFlip)(),
+        LazyCall(transforms.ToTensor)(),
+        LazyCall(transforms.Normalize)(mean=CIFAR100_TRAIN_MEAN, std=CIFAR100_TRAIN_STD),
+    ]
+)
+
+test_aug = LazyCall(transforms.Compose)(
+    transforms=[
+        LazyCall(transforms.Resize)(
+            size=256,
+            interpolation=InterpolationMode.BICUBIC,
+        ),
+        LazyCall(transforms.CenterCrop)(
+            size=224,
+        ),
+        LazyCall(transforms.ToTensor)(),
+        LazyCall(transforms.Normalize)(
+            mean=CIFAR100_TRAIN_MEAN,
+            std=CIFAR100_TRAIN_STD,
+        ),
+    ]
+)
+
+
+# Dataloader config
+dataloader = OmegaConf.create()
+dataloader.train = LazyCall(build_image_train_loader)(
+    dataset=[
+        LazyCall(CIFAR100Dataset)(
+            root="./",
+            train=True,
+            download=True,
+            transform=train_aug,
+        ),
+    ],
+    num_workers=4,
+    mixup_func=LazyCall(Mixup)(
+        mixup_alpha=0.8,
+        cutmix_alpha=1.0,
+        prob=1.0,
+        switch_prob=0.5,
+        mode="batch",
+        num_classes=100,
+    ),
+)
+
+dataloader.test = [
+    LazyCall(build_image_test_loader)(
+        dataset=LazyCall(CIFAR100Dataset)(
+            root="./",
+            train=False,
+            download=True,
+            transform=test_aug,
+        ),
+        num_workers=4,
+    )
+]

configs/common/data/gpt_dataset.py

+from libai.config import LazyCall
+from omegaconf import OmegaConf
+from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader
+from libai.data.datasets import GPT2Dataset
+from libai.data.data_utils import get_indexed_dataset
+
+from libai.tokenizer import GPT2Tokenizer
+
+
+tokenization = OmegaConf.create()
+
+tokenization.tokenizer = LazyCall(GPT2Tokenizer)(
+    vocab_file="/workspace/data/gpt_dataset/gpt2-vocab.json",
+    merges_file="/workspace/data/gpt_dataset/gpt2-merges.txt",
+    do_lower_case=True,
+    do_chinese_wwm=True,
+)
+tokenization.append_eod = False
+tokenization.make_vocab_size_divisible_by = 128
+
+dataloader = OmegaConf.create()
+
+dataloader.train = LazyCall(build_nlp_train_val_test_loader)(
+    dataset=[
+        LazyCall(GPT2Dataset)(
+            name="gpt-2",
+            data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+            indexed_dataset=LazyCall(get_indexed_dataset)(
+                data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+                data_impl="mmap",
+                skip_warmup=False,
+            ),
+            max_seq_length=1024,
+            seed=1234,
+        ),
+    ],
+    train_val_test_num_samples=None,  # a hint for deferred assignment
+    splits=[[949.0, 50.0, 1.0]],
+    weights=[1.0],
+    num_workers=4,
+)

configs/common/data/imagenet.py

+from omegaconf import OmegaConf
+from flowvision import transforms
+from flowvision.transforms import InterpolationMode
+from flowvision.transforms.functional import str_to_interp_mode
+from flowvision.data.constants import (
+    IMAGENET_DEFAULT_MEAN,
+    IMAGENET_DEFAULT_STD,
+)
+from flowvision.data.auto_augment import rand_augment_transform
+from flowvision.data.random_erasing import RandomErasing
+
+from libai.config import LazyCall
+from libai.data.datasets import ImageNetDataset
+from libai.data.build import build_image_train_loader, build_image_test_loader
+
+train_aug = LazyCall(transforms.Compose)(
+    transforms=[
+        LazyCall(transforms.RandomResizedCrop)(
+            size=224,
+            scale=(0.08, 1.0),
+            ratio=(3.0 / 4.0, 4.0 / 3.0),
+            interpolation=InterpolationMode.BICUBIC,
+        ),
+        LazyCall(transforms.RandomHorizontalFlip)(p=0.5),
+        LazyCall(rand_augment_transform)(
+            config_str="rand-m9-mstd0.5-inc1",
+            hparams=dict(
+                translate_const=int(224 * 0.45),
+                img_mean=tuple([min(255, round(255 * x)) for x in IMAGENET_DEFAULT_MEAN]),
+                interpolation=str_to_interp_mode("bicubic"),
+            ),
+        ),
+        LazyCall(transforms.ToTensor)(),
+        LazyCall(transforms.Normalize)(
+            mean=IMAGENET_DEFAULT_MEAN,
+            std=IMAGENET_DEFAULT_STD,
+        ),
+        LazyCall(RandomErasing)(
+            probability=0.25,
+            mode="pixel",
+            max_count=1,
+            num_splits=0,
+            device="cpu",
+        ),
+    ]
+)
+
+
+test_aug = LazyCall(transforms.Compose)(
+    transforms=[
+        LazyCall(transforms.Resize)(
+            size=256,
+            interpolation=InterpolationMode.BICUBIC,
+        ),
+        LazyCall(transforms.CenterCrop)(
+            size=224,
+        ),
+        LazyCall(transforms.ToTensor)(),
+        LazyCall(transforms.Normalize)(
+            mean=IMAGENET_DEFAULT_MEAN,
+            std=IMAGENET_DEFAULT_STD,
+        ),
+    ]
+)
+
+
+dataloader = OmegaConf.create()
+dataloader.train = LazyCall(build_image_train_loader)(
+    dataset=[
+        LazyCall(ImageNetDataset)(
+            root="./dataset",
+            train=True,
+            transform=train_aug,
+        ),
+    ],
+    num_workers=4,
+    mixup_func=None,
+)
+
+
+dataloader.test = [
+    LazyCall(build_image_test_loader)(
+        dataset=LazyCall(ImageNetDataset)(
+            root="./dataset",
+            train=False,
+            transform=test_aug,
+        ),
+        num_workers=4,
+    )
+]

configs/common/data/roberta_dataset.py

+from libai.config import LazyCall
+from omegaconf import OmegaConf
+from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader
+from libai.data.datasets import RobertaDataset
+from libai.data.data_utils import get_indexed_dataset
+
+from libai.tokenizer import RobertaTokenizer
+
+
+tokenization = OmegaConf.create()
+
+tokenization.tokenizer = LazyCall(RobertaTokenizer)(
+    vocab_file="roberta-vocab.json", merges_file="roberta-merges.txt"
+)
+tokenization.append_eod = False
+tokenization.make_vocab_size_divisible_by = 128
+
+dataloader = OmegaConf.create()
+
+dataloader.train = LazyCall(build_nlp_train_val_test_loader)(
+    dataset=[
+        LazyCall(RobertaDataset)(
+            name="roberta",
+            data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+            indexed_dataset=LazyCall(get_indexed_dataset)(
+                data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+                data_impl="mmap",
+                skip_warmup=False,
+            ),
+            max_seq_length=512,
+            mask_lm_prob=0.15,
+            short_seq_prob=0.0,
+            seed=1234,
+            masking_style="bert",
+        ),
+    ],
+    train_val_test_num_samples=None,  # a hint for deferred assignment
+    splits=[[949.0, 50.0, 1.0]],
+    weights=[1.0],
+    num_workers=4,
+)

configs/common/data/t5_dataset.py

+from libai.config import LazyCall
+from omegaconf import OmegaConf
+from libai.data import build_nlp_test_loader, build_nlp_train_val_test_loader
+from libai.data.datasets import T5Dataset
+from libai.data.data_utils import get_indexed_dataset
+
+from libai.tokenizer import BertTokenizer
+
+
+tokenization = OmegaConf.create()
+
+tokenization.setup = True
+
+special_tokens = []
+for i in range(100):
+    special_tokens.append(f"<extra_id_{i}>")
+tokenization.tokenizer = LazyCall(BertTokenizer)(
+    vocab_file="/workspace/data/libai_dataset/bert-base-chinese-vocab.txt",
+    do_lower_case=True,
+    do_chinese_wwm=True,
+    bos_token="[BOS]",
+    eos_token="[EOS]",
+    additional_special_tokens=special_tokens,
+)
+tokenization.append_eod = False
+tokenization.make_vocab_size_divisible_by = 128
+
+dataloader = OmegaConf.create()
+
+dataloader.train = LazyCall(build_nlp_train_val_test_loader)(
+    dataset=[
+        LazyCall(T5Dataset)(
+            name="t5",
+            data_prefix="/workspace/data/libai_dataset/loss_compara_content_sentence",
+            indexed_dataset=LazyCall(get_indexed_dataset)(
+                data_prefix="/workspace/data/libai_dataset/" "/loss_compara_content_sentence",
+                data_impl="mmap",
+                skip_warmup=False,
+            ),
+            max_seq_length=512,
+            max_seq_length_dec=128,
+            masked_lm_prob=0.15,
+            short_seq_prob=0.1,
+            seed=1234,
+        ),
+    ],
+    train_val_test_num_samples=None,  # a hint for deferred assignment
+    splits=[[949.0, 50.0, 1.0]],
+    weights=[1.0],
+    num_workers=4,
+)

configs/common/models/bert.py

+from omegaconf import DictConfig
+from libai.config import LazyCall
+from libai.models import BertModel, BertForPreTraining
+
+
+cfg = dict(
+    vocab_size=30522,
+    hidden_size=768,
+    hidden_layers=24,
+    num_attention_heads=12,
+    intermediate_size=4096,
+    hidden_dropout_prob=0.1,
+    attention_probs_dropout_prob=0.1,
+    max_position_embeddings=512,
+    num_tokentypes=2,
+    add_pooling_layer=True,
+    initializer_range=0.02,
+    layernorm_eps=1e-5,
+    bias_gelu_fusion=True,
+    bias_dropout_fusion=True,
+    scale_mask_softmax_fusion=True,
+    apply_query_key_layer_scaling=True,
+    apply_residual_post_layernorm=False,
+    add_binary_head=True,
+    amp_enabled=False,
+)
+
+cfg = DictConfig(cfg)
+
+bert_model = LazyCall(BertModel)(cfg=cfg)
+
+pretrain_model = LazyCall(BertForPreTraining)(cfg=cfg)

configs/common/models/gpt.py

+from omegaconf import DictConfig
+from libai.config import LazyCall
+from libai.models import GPTModel, GPTForPreTraining
+
+
+cfg = dict(
+    hidden_layers=6,
+    vocab_size=30522,
+    hidden_size=384,
+    ffn_hidden_size=1536,
+    num_attention_heads=12,
+    max_seq_length=1024,
+    embedding_dropout_prob=0,
+    attention_dropout_prob=0,
+    output_dropout_prob=0,
+    layernorm_epsilon=1e-5,
+    initializer_range=0.02,
+    use_scaled_init_for_output_weights=True,
+    bias_gelu_fusion=True,
+    bias_dropout_fusion=True,
+    scale_mask_softmax_fusion=True,
+    apply_query_key_layer_scaling=True,
+    apply_residual_post_layernorm=False,
+    amp_enabled=False,
+)
+
+cfg = DictConfig(cfg)
+
+gpt_model = LazyCall(GPTModel)(cfg=cfg)
+
+pretrain_model = LazyCall(GPTForPreTraining)(cfg=cfg)

configs/common/models/graph.py

+from omegaconf import DictConfig
+from libai.config import LazyCall
+from libai.models.utils import GraphBase
+
+
+graph = dict(
+    # options for graph or eager mode
+    enabled=True,
+    debug=-1,  # debug mode for graph
+    auto_parallel=dict(
+        enabled=False,
+        enable_auto_parallel_ignore_user_sbp_config=False,  # ignore all .to_global() in graph
+        trunk_algo=True,  # consider overlapping calculate time and transfer time
+        sbp_collector=False,  # use proxy node when one node transfer to many nodes
+    ),
+    train_graph=LazyCall(GraphBase)(
+        is_train=True,
+    ),
+    eval_graph=LazyCall(GraphBase)(is_train=False),
+)
+
+graph = DictConfig(graph)

configs/common/models/resmlp/resmlpB_24.py

+from libai.config import LazyCall
+from libai.models import ResMLP
+
+from .resmlp_12 import cfg
+
+
+cfg.patch_size = 8
+cfg.embed_dim = 768
+cfg.depth = 24
+cfg.init_scale = 1e-6
+
+model = LazyCall(ResMLP)(cfg=cfg)

configs/common/models/resmlp/resmlp_12.py

+from omegaconf import DictConfig
+from libai.config import LazyCall
+from libai.models import ResMLP
+
+
+cfg = dict(
+    img_size=224,
+    patch_size=16,
+    in_chans=3,
+    embed_dim=384,
+    depth=12,
+    drop_rate=0.0,
+    drop_path_rate=0.05,
+    init_scale=0.1,
+    num_classes=1000,
+    loss_func=None,
+)
+
+cfg = DictConfig(cfg)
+
+model = LazyCall(ResMLP)(cfg=cfg)

configs/common/models/resmlp/resmlp_24.py

+from libai.config import LazyCall
+from libai.models import ResMLP
+
+from .resmlp_12 import cfg
+
+
+cfg.depth = 24
+cfg.init_scale = 1e-5
+
+model = LazyCall(ResMLP)(cfg=cfg)

configs/common/models/resmlp/resmlp_36.py

+from libai.config import LazyCall
+from libai.models import ResMLP
+
+from .resmlp_12 import cfg
+
+
+cfg.depth = 36
+cfg.init_scale = 1e-6
+
+model = LazyCall(ResMLP)(cfg=cfg)

configs/common/models/roberta.py

+from omegaconf import DictConfig
+from libai.config import LazyCall
+from libai.models import RobertaModel, RobertaForPreTraining, RobertaForCausalLM
+
+
+cfg = dict(
+    vocab_size=50265,
+    hidden_size=768,
+    hidden_layers=12,
+    num_attention_heads=12,
+    intermediate_size=3072,
+    hidden_dropout_prob=0.1,
+    attention_probs_dropout_prob=0.1,
+    max_position_embeddings=514,
+    num_tokentypes=1,
+    add_pooling_layer=True,
+    initializer_range=0.02,
+    layernorm_eps=1e-5,
+    pad_token_id=1,
+    bias_gelu_fusion=True,
+    bias_dropout_fusion=True,
+    scale_mask_softmax_fusion=True,
+    apply_query_key_layer_scaling=True,
+    apply_residual_post_layernorm=False,
+    amp_enabled=False,
+)
+
+cfg = DictConfig(cfg)
+
+roberta_model = LazyCall(RobertaModel)(cfg=cfg)
+
+roberta_causal_lm = LazyCall(RobertaForCausalLM)(cfg=cfg)
+
+pretrain_model = LazyCall(RobertaForPreTraining)(cfg=cfg)