gpt2

dev/model_test.sh

+#!/bin/bash -e
+
+# cd to libai project root
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+export TEST_OUTPUT=output_unittest
+export ONEFLOW_TEST_DEVICE_NUM=4
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_bert.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_roberta.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_gpt.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_t5.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_mt5.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_vit.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_swin.py
+
+python3 -m oneflow.distributed.launch --nproc_per_node 4 -m pytest -s --disable-warnings tests/models/test_swinv2.py
+
+rm -rf $TEST_OUTPUT

dev/run_unittest.sh

+#!/bin/bash -e
+
+# cd to libai project root
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+pytest --disable-warnings ./tests

docker/Dockerfile

+FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04
+# use an older system (18.04) to avoid opencv incompatibility (issue#3524)
+
+# Comment it if you are not in China
+RUN sed -i 's/security.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list
+RUN sed -i 's/archive.ubuntu.com/mirrors.ustc.edu.cn/g' /etc/apt/sources.list
+
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install -y \
+	python3-opencv ca-certificates python3-dev git wget sudo ninja-build
+RUN ln -sv /usr/bin/python3 /usr/bin/python
+
+# create a non-root user
+ARG USER_ID=1000
+RUN useradd -m --no-log-init --system  --uid ${USER_ID} appuser -g sudo
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER appuser
+WORKDIR /home/appuser
+
+ENV PATH="/home/appuser/.local/bin:${PATH}"
+RUN wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \
+	python3 get-pip.py --user && \
+	rm get-pip.py
+
+# install dependencies
+# See https://pytorch.org/ for other options if you use a different version of CUDA
+RUN pip install --user pybind11 cmake   # cmake from apt-get is too old
+RUN python3 -m pip install --user --pre oneflow -f https://staging.oneflow.info/branch/master/cu112
+
+# install libai
+RUN git clone https://github.com/Oneflow-Inc/libai.git libai_repo
+
+RUN pip install -i https://pypi.tuna.tsinghua.edu.cn/simple --user -e  libai_repo
+
+WORKDIR /home/appuser/libai_repo

docker/README.md

+## Use the container (with docker >= 19.03)
+
+```shell
+cd docker/
+# Build:
+docker build --build-arg USER_ID=$UID -t libai:v0 .
+# Launch (require GPUs):
+docker run --gpus all -it --name libai -v /home/user/:/workspace/ --ipc host --net host libai:v0 /bin/bash
+```
+
+You can also use the pre-built LiBai containers (the latest compatible version at time of publication can be pulled with `docker pull l1aoxingyu/libai:v0`).
+
+> NOTE: If your nvidia-driver cannot support `cuda111`, you can build the base image from `cuda102`.
+
+## Install new dependencies
+
+Add the following to `Dockerfile` to make persistent changes.
+
+```shell
+RUN sudo apt-get update && sudo apt-get install -y vim
+```
+
+Or run them in the container to make temporary changes.

docs/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+html: Makefile
+	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+clean: Makefile
+	@rm -rf build

docs/README.md

+## Read LiBai Documentation
+The latest documentation built from this directory is available at [libai.readthedocs.io](https://libai.readthedocs.io/).
+
+
+## Build LiBai Documentation
+1. Install LiBai according to [Installation](https://libai.readthedocs.io/en/latest/tutorials/get_started/Installation.html).
+2. Install additional libraries and run `make html` for building the docs:
+```bash
+cd ${libai-path}/docs
+pip install -r requirements.txt --user
+make html
+```

docs/requirements.txt

+sphinx==3.5.4
+jinja2<3.1
+recommonmark==0.6.0
+sphinx-rtd-theme==1.0.0
+git+https://github.com/Oneflow-Inc/libai.git
+--pre
+-f https://staging.oneflow.info/branch/master/cpu
+oneflow

docs/source/_static/css/line_space.css

+.rst-content .section ol li>*, .rst-content .section ul li>* {
+  margin-top: 0px;
+  margin-bottom: 0px;
+}
+
+.rst-content .section ol li>*, .rst-content .section ul li ul {
+  margin-top: 0px;
+  margin-bottom: 0px;
+}

docs/source/_templates/line_space.html

+{% extends "!line_space.html" %}
+{% set css_files = css_files + [ "_static/css/line_space.css" ] %}

docs/source/changelog.md

+## Changelog
+
+### v0.2.0 (07/07/2022)
+**New Features:**
+- Support evaluation enabled and set `eval_iter`
+- Support customized sampler in `config.py`
+- Support rdma for pipeline-model-parallel
+- Support multi fused kernel 
+   - fused_scale_mask_softmax_dropout
+   - fused_scale_tril_softmax_mask_scale
+   - fused_self_attention in branch `libai_bench`
+- User Experience Optimization
+- Optimization for training throughput, see [benchmark](https://libai.readthedocs.io/en/latest/tutorials/get_started/Benchmark.html) for more details
+
+**New Supported Models:**
+
+- Support 3D parallel [Roberta](https://arxiv.org/abs/1907.11692) model
+- Support 2D parallel (data parallel + tensor model parallel) [SimCSE](https://arxiv.org/abs/2104.08821) model
+- Support Data parallel [MAE](https://arxiv.org/abs/2111.06377) model
+- Support Data parallel [MOCOV3](https://arxiv.org/abs/2104.02057) model
+
+### v0.1.0 (22/03/2022)
+
+**New Features:**
+- Support Data Parallelism
+- Support 1D Tensor Parallelism
+- Support Pipeline Parallelism
+- Unified distributed Layers for both single-GPU and multi-GPU training
+- `LazyConfig` system for more flexible syntax and no predefined structures
+- Easy-to-use trainer and engine
+- Support both CV and NLP data processing
+- Mixed Precision Training
+- Activation Checkpointing
+- Gradient Accumulation
+- Gradient Clipping
+- Zero Redundancy Optimizer (ZeRO)
+
+**Supported Models:**
+- Support 3D parallel [BERT](https://arxiv.org/abs/1810.04805) model
+- Support 3D parallel [GPT-2](https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf) model
+- Support 3D parallel [T5](https://arxiv.org/abs/1910.10683) model
+- Support 3D parallel [Vision Transformer](https://arxiv.org/abs/2010.11929) model
+- Support Data parallel [Swin Transformer](https://arxiv.org/abs/2103.14030) model
+- Support finetune task in [QQP project](https://github.com/Oneflow-Inc/libai/tree/main/projects/QQP)
+- Support text classification task in [text classification project](https://github.com/Oneflow-Inc/libai/tree/main/projects/text_classification)
+

docs/source/conf.py

+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+import libai  # noqa
+
+sys.path.insert(0, os.path.abspath("."))
+
+# -- Project information -----------------------------------------------------
+
+project = "libai"
+copyright = "2021, OneFlow"
+author = "OneFlow"
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "recommonmark",
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = "Python"
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# html_theme = "furo"
+html_theme = "sphinx_rtd_theme"
+
+
+def setup(app):
+    app.add_css_file("css/line_space.css")
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]

docs/source/index.rst

+.. libai documentation master file, created by
+   sphinx-quickstart on Mon Nov 29 10:26:07 2021.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to libai's documentation!
+======================================
+
+.. toctree::
+   :maxdepth: 3
+
+   tutorials/index
+   notes/index
+   modules/index
+   changelog.md
\ No newline at end of file

docs/source/modules/index.rst

+API Documentation
+=================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: API Documentation
+
+   libai.config
+   libai.data
+   libai.engine
+   libai.evaluation
+   libai.layers
+   libai.models
+   libai.optim
+   libai.scheduler
+   libai.tokenizer
+   libai.utils
\ No newline at end of file

docs/source/modules/libai.config.rst

+libai.config
+##############################
+
+.. currentmodule:: libai.config
+.. automodule:: libai.config
+    :members: 
+        try_get_key,
+        get_config,
+        configurable,
+        instantiate,
+        LazyCall,
+        LazyConfig,

docs/source/modules/libai.data.rst

+libai.data
+##############################
+
+libai.data.data_utils module
+---------------------------------
+.. currentmodule:: libai.data
+.. automodule:: libai.data.data_utils
+    :members:
+        IndexedCachedDataset,
+        IndexedDataset,
+        MMapIndexedDataset,
+        get_indexed_dataset,
+
+
+libai.data.datasets module
+---------------------------------
+.. currentmodule:: libai.data
+.. automodule:: libai.data.datasets
+    :members: 
+        MNISTDataset,
+        CIFAR10Dataset,
+        CIFAR100Dataset,
+        ImageNetDataset,
+        BertDataset,
+        RobertaDataset,
+        T5Dataset,
+        GPT2Dataset
+
+
+libai.data.samplers module
+---------------------------------
+.. currentmodule:: libai.data
+.. automodule:: libai.data.samplers
+    :members:
+        CyclicSampler,
+        SingleRoundSampler,
+
+libai.data.build module
+---------------------------------
+.. currentmodule:: libai.data
+.. automodule:: libai.data.build
+    :members:
+        build_nlp_train_val_test_loader,
+        build_nlp_train_loader,
+        build_nlp_test_loader,
+        build_image_train_loader,
+        build_image_test_loader,
+        

docs/source/modules/libai.engine.rst

+libai.engine
+##############################
+
+libai.engine.default module
+---------------------------------
+.. currentmodule:: libai.engine
+.. automodule:: libai.engine.default
+    :member-order: bysource
+    :members: 
+        default_setup,
+        DefaultTrainer,
+
+
+libai.engine.hooks module
+---------------------------------
+.. currentmodule:: libai.engine
+.. automodule:: libai.engine.hooks
+    :member-order: bysource
+    :members: 
+        CallbackHook,
+        IterationTimer,
+        PeriodicWriter,
+        PeriodicCheckpointer,
+        BestCheckpointer,
+        EvalHook,
+        LRScheduler,
+
+
+libai.engine.trainer module
+---------------------------------
+.. currentmodule:: libai.engine
+.. automodule:: libai.engine.trainer
+    :member-order: bysource
+    :members: 
+        HookBase,
+        TrainerBase,
+        EagerTrainer,
+        GraphTrainer,
\ No newline at end of file

docs/source/modules/libai.evaluation.rst

+libai.evaluation
+##############################
+
+.. currentmodule:: libai.evaluation
+.. automodule:: libai.evaluation
+    :members: 
+        DatasetEvaluator,
+        inference_on_dataset,
+        print_csv_format,
+        flatten_results_dict,
+        ClsEvaluator,
+        PPLEvaluator,
+        RegEvaluator,
+        BLEUEvaluator,

docs/source/modules/libai.layers.rst

+libai.layers
+##############################
+
+.. currentmodule:: libai.layers
+.. automodule:: libai.layers
+    :members: 
+        Embedding,
+        VocabEmbedding,
+        SinePositionalEmbedding,
+        PatchEmbedding,
+        drop_path,
+        DropPath,
+        build_activation,
+        Linear,
+        Linear1D,
+        MLP,
+        LayerNorm,
+        RMSLayerNorm,
+        MultiheadAttention,
+        TransformerLayer,
+        ParallelCrossEntropyLoss
\ No newline at end of file

docs/source/modules/libai.models.rst

+libai.models
+##############################
+Supported models in LiBai(李白)
+
+- `VisionTransformer`_
+- `SwinTransformer`_
+- `SwinTransformerV2`_
+- `ResMLP`_
+- `BERT`_
+- `RoBERTa`_
+- `T5`_
+- `GPT-2`_
+
+.. _VisionTransformer: https://arxiv.org/abs/2010.11929
+.. _SwinTransformer: https://arxiv.org/abs/2103.14030
+.. _SwinTransformerV2: https://arxiv.org/abs/2111.09883
+.. _ResMLP: https://arxiv.org/abs/2105.03404
+.. _BERT: https://arxiv.org/abs/1810.04805
+.. _RoBERTa: https://arxiv.org/abs/1907.11692
+.. _T5: https://arxiv.org/abs/1910.10683
+.. _GPT-2: https://cdn.openai.com/better-language-models/language_models_are_unsupervised_multitask_learners.pdf
+
+
+
+.. currentmodule:: libai.models
+
+.. automodule:: libai.models.build
+    :members:
+        build_model,
+        build_graph,
+
+VisionTransformer
+-----------------
+.. automodule:: libai.models.vision_transformer
+    :members: 
+        VisionTransformer,
+
+SwinTransformer
+---------------
+.. automodule:: libai.models.swin_transformer
+    :members: 
+        SwinTransformer,
+
+SwinTransformerV2
+-----------------
+.. automodule:: libai.models.swin_transformer_v2
+    :members:
+        SwinTransformerV2,
+
+ResMLP
+-------
+.. automodule:: libai.models.resmlp
+    :members:
+        ResMLP,
+
+BERT
+----
+.. automodule:: libai.models.bert_model
+    :members: 
+        BertModel,
+        BertForPreTraining,
+
+RoBERTa
+-------
+.. automodule:: libai.models.roberta_model
+    :members: 
+        RobertaModel,
+        RobertaForPreTraining,
+        RobertaForCausalLM,
+
+T5
+---
+.. automodule:: libai.models.t5_model
+    :members: 
+        T5ForPreTraining,
+        T5Model,
+
+GPT-2
+-----
+.. automodule:: libai.models.gpt_model
+    :members: 
+        GPTForPreTraining,
+        GPTModel
\ No newline at end of file