Oneflow 0.8 for DCU

docs/source/functional.rst

+oneflow.nn.functional
+===========================================
+Functional operations for neural networks
+-------------------------------------------
+.. currentmodule:: oneflow.nn.functional
+.. autofunction:: conv1d
+.. autofunction:: conv2d
+.. autofunction:: conv3d
+.. autofunction:: conv_transpose1d
+.. autofunction:: conv_transpose2d
+.. autofunction:: conv_transpose3d
+.. autofunction:: adaptive_avg_pool1d
+.. autofunction:: adaptive_avg_pool2d
+.. autofunction:: adaptive_avg_pool3d
+.. autofunction:: relu
+.. autofunction:: hardsigmoid
+.. autofunction:: hardshrink
+.. autofunction:: hardswish
+.. autofunction:: hardtanh
+.. autofunction:: normalize
+.. autofunction:: layer_norm
+.. autofunction:: leaky_relu
+.. autofunction:: elu
+.. autofunction:: celu
+.. autofunction:: selu
+.. autofunction:: sigmoid
+.. autofunction:: pad
+.. autofunction:: prelu
+.. autofunction:: logsigmoid 
+.. autofunction:: log_softmax
+.. autofunction:: gelu
+.. autofunction:: glu
+.. autofunction:: softsign
+.. autofunction:: softmax 
+.. autofunction:: softplus
+.. autofunction:: tanh 
+.. autofunction:: threshold
+.. autofunction:: softshrink 
+.. autofunction:: silu
+.. autofunction:: mish
+.. autofunction:: one_hot
+.. autofunction:: triplet_margin_loss
+.. autofunction:: dropout 
+.. autofunction:: affine_grid
+.. autofunction:: grid_sample
+.. autofunction:: interpolate
+.. autofunction:: ctc_greedy_decoder
+.. autofunction:: sparse_softmax_cross_entropy
+.. autofunction:: embedding
+.. autofunction:: linear
+.. autofunction:: cosine_similarity
+.. autofunction:: cross_entropy

docs/source/graph.rst

+oneflow.nn.Graph
+============================================================
+Base class for running neural networks in Static Graph Mode.
+------------------------------------------------------------
+.. currentmodule:: oneflow.nn
+.. autoclass:: oneflow.nn.Graph
+    :members: __init__,
+            build,
+            __call__,
+            add_optimizer,
+            set_grad_scaler,
+            state_dict,
+            load_state_dict,
+            name,
+            debug,
+            __repr__,
+    :member-order: bysource
+
+
+
+.. autoclass:: oneflow.nn.graph.graph_config.GraphConfig
+    :members: enable_amp,
+            enable_zero,
+            allow_fuse_model_update_ops,
+            allow_fuse_add_to_output,
+            allow_fuse_cast_scale,
+            set_gradient_accumulation_steps,
+            enable_cudnn_conv_heuristic_search_algo,
+            enable_straighten_algorithm,
+    :member-order: bysource
+
+
+
+.. autoclass:: oneflow.nn.graph.block_config.BlockConfig
+    :members: stage_id,
+            set_stage,
+            activation_checkpointing,
+    :member-order: bysource
+

docs/source/image.rst

+oneflow.nn.image
+======================================
+Image operations for neural networks
+--------------------------------------
+.. currentmodule:: oneflow.nn.image
+.. automodule:: oneflow.nn.image
+    :members: Resize,
+        batch_align,
+        decode,
+        flip,
+        normalize

docs/source/index.rst

+OneFlow API Reference
+===================================
+.. toctree::
+    :maxdepth: 1
+
+    troubleshooting
+
+
+
+.. toctree::
+    :maxdepth: 1
+    :caption: OneFlow Python API
+
+    oneflow
+    tensor
+    tensor_attributes
+    nn
+    functional
+    autograd
+    cuda
+    distributed
+    linalg
+    nn.init
+    optim
+    module
+    graph
+    image
+    utils
+    env
+    comm
+    one_embedding
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/linalg.rst

+oneflow.linalg
+===================================
+OneFlow linear algebra operations.
+----------------------------------
+.. currentmodule:: oneflow.linalg
+.. autofunction:: oneflow.linalg.matrix_norm
+.. autofunction:: oneflow.linalg.norm       
+.. autofunction:: oneflow.linalg.vector_norm

docs/source/module.rst

+oneflow.nn.Module
+================================================
+Module class for building neural networks
+---------------------------------------------------
+.. currentmodule:: oneflow.nn
+.. autoclass:: oneflow.nn.Module
+    :members:
+

docs/source/nn.init.rst

+oneflow.nn.init
+===================================
+Operators for initialization
+----------------------------------
+.. currentmodule:: oneflow.nn.init
+
+.. autofunction:: oneflow.nn.init.xavier_uniform_
+.. autofunction:: oneflow.nn.init.xavier_normal_
+.. autofunction:: oneflow.nn.init.kaiming_uniform_
+.. autofunction:: oneflow.nn.init.kaiming_normal_
+.. autofunction:: oneflow.nn.init.orthogonal_

docs/source/nn.rst

+oneflow.nn
+===================================
+Operators for neural networks
+----------------------------------
+.. currentmodule:: oneflow.nn
+.. automodule:: oneflow.nn
+    :members: AdaptiveAvgPool1d,
+        AdaptiveAvgPool2d,
+        AdaptiveAvgPool3d,
+        AvgPool1d,
+        AvgPool2d,
+        AvgPool3d,
+        BCELoss,
+        BCEWithLogitsLoss,
+        BatchNorm1d,
+        BatchNorm2d,
+        BatchNorm3d,
+        COCOReader,
+        CTCLoss,
+        CoinFlip,
+        ConstantPad1d,
+        ConstantPad2d,
+        ConstantPad3d,
+        Conv1d,
+        Conv2d,
+        Conv3d,
+        ConvTranspose1d,
+        ConvTranspose2d,
+        ConvTranspose3d,
+        CosineSimilarity,
+        CombinedMarginLoss,
+        CropMirrorNormalize,
+        CrossEntropyLoss,
+        Dropout,
+        ELU,
+        CELU,
+        Embedding,
+        Flatten,
+        GELU,
+        GLU,
+        GroupNorm,
+        Hardsigmoid,
+        Hardshrink,
+        Hardswish,
+        Hardtanh,
+        Identity,
+        InstanceNorm1d,
+        InstanceNorm2d,
+        InstanceNorm3d,
+        KLDivLoss,
+        L1Loss,
+        LayerNorm,
+        LeakyReLU,
+        Linear,
+        LogSigmoid,
+        LogSoftmax,
+        MSELoss,
+        MarginRankingLoss,
+        TripletMarginLoss,
+        MaxPool1d,
+        MaxPool2d,
+        MaxPool3d,
+        ModuleDict,
+        ModuleList,
+        Mish,
+        NLLLoss,
+        OFRecordImageDecoder,
+        OFRecordImageDecoderRandomCrop,
+        OFRecordRawDecoder,
+        OFRecordReader,
+        OFRecordBytesDecoder,
+        PReLU,
+        Parameter,
+        ParameterDict,
+        ParameterList,
+        PixelShuffle,
+        ReLU,
+        ReLU6,
+        ReflectionPad2d,
+        ReplicationPad2d,
+        Sequential, 
+        SELU, 
+        SiLU, 
+        Sigmoid,
+        SmoothL1Loss,
+        Softmax,
+        Softplus, 
+        Softshrink,
+        Softsign, 
+        Tanh,
+        Threshold,
+        Upsample,
+        UpsamplingBilinear2d,
+        UpsamplingNearest2d,
+        ZeroPad2d,
+        MinMaxObserver,
+        MovingAverageMinMaxObserver,
+        FakeQuantization,
+        Quantization, 
+        FusedBatchNorm1d, 
+        FusedBatchNorm2d, 
+        FusedBatchNorm3d, 
+        FusedMLP, 
+
+.. autofunction:: oneflow.nn.modules.pixelshuffle.PixelShufflev2
+
+.. autofunction:: oneflow.nn.parallel.DistributedDataParallel
+
+.. currentmodule:: oneflow.nn.utils
+.. autofunction:: oneflow.nn.utils.clip_grad_norm_
+.. autofunction:: oneflow.nn.utils.weight_norm
+.. autofunction:: oneflow.nn.utils.remove_weight_norm

docs/source/one_embedding.rst

+oneflow.one_embedding
+===================================
+OneFlow one_embedding operations.
+----------------------------------
+.. currentmodule:: oneflow.one_embedding
+.. autoclass:: MultiTableEmbedding
+    :members: forward,
+              save_snapshot,
+              load_snapshot,
+.. autofunction:: oneflow.one_embedding.MultiTableEmbedding.forward
+.. autoclass:: MultiTableMultiColumnEmbedding
+    :members: forward,
+              save_snapshot,
+              load_snapshot,
+.. autofunction:: oneflow.one_embedding.MultiTableMultiColumnEmbedding.forward
+.. autofunction:: oneflow.one_embedding.make_device_mem_store_options
+.. autofunction:: oneflow.one_embedding.make_cached_ssd_store_options       
+.. autofunction:: oneflow.one_embedding.make_cached_host_mem_store_options
+.. autofunction:: oneflow.one_embedding.make_uniform_initializer
+.. autofunction:: oneflow.one_embedding.make_normal_initializer
+.. autofunction:: oneflow.one_embedding.make_table_options
+.. autofunction:: oneflow.one_embedding.make_table
+.. automodule:: oneflow.one_embedding
+    :members: Ftrl
+.. autofunction:: oneflow.one_embedding.make_persistent_table_reader
+.. autofunction:: oneflow.one_embedding.make_persistent_table_writer

docs/source/oneflow.rst

+oneflow
+===================================
+oneflow
+----------------------------------
+.. currentmodule:: oneflow
+.. automodule:: oneflow
+    :members: adaptive_avg_pool1d, 
+            adaptive_avg_pool2d, 
+            adaptive_avg_pool3d, 
+            abs, 
+            acos, 
+            acosh, 
+            add, 
+            addcmul, 
+            addmm, 
+            all, 
+            amin, 
+            amax,
+            any, 
+            arccos, 
+            arcsin, 
+            arcsinh, 
+            arccosh, 
+            arctan, 
+            arctanh, 
+            argmax, 
+            argmin, 
+            arange, 
+            argsort, 
+            argwhere,
+            asin,  
+            asinh, 
+            atan, 
+            atan2, 
+            atanh, 
+            bernoulli,
+            broadcast_like, 
+            batch_gather,
+            bmm,
+            cat, 
+            concat,
+            cast, 
+            ceil, 
+            chunk, 
+            clamp, 
+            clip, 
+            cos, 
+            cosh, 
+            diag, 
+            select,
+            diagonal,
+            movedim,
+            tensor_split,
+            hsplit,
+            vsplit,
+            as_strided,
+            div, 
+            dot, 
+            eq,
+            einsum,
+            equal, 
+            expand, 
+            eye,
+            exp, 
+            expm1, 
+            erf, 
+            erfc, 
+            erfinv, 
+            flatten, 
+            flip, 
+            floor, 
+            floor_,
+            fmod,
+            full, 
+            gather, 
+            gather_nd, 
+            gelu, 
+            gt, 
+            in_top_k, 
+            index_select,
+            linspace,
+            logical_and,
+            logical_or,
+            logical_not,
+            logical_xor,
+            load, 
+            log, 
+            log2,
+            log1p, 
+            lt, 
+            le, 
+            masked_fill, 
+            masked_select, 
+            matmul, 
+            mm, 
+            mv, 
+            narrow, 
+            max, 
+            mean,
+            median,
+            mish,  
+            min, 
+            meshgrid,
+            mul, 
+            neg, 
+            negative, 
+            new_ones,
+            nonzero,
+            normal,
+            numel, 
+            ne, 
+            empty,
+            ones, 
+            ones_like, 
+            pow,
+            prod,  
+            rand,
+            randn,
+            repeat, 
+            repeat_interleave,
+            reshape, 
+            randint,
+            randperm,
+            reciprocal,
+            roc_auc_score,
+            roll,
+            round, 
+            rsqrt,
+            save, 
+            scatter,
+            scatter_add,
+            scatter_nd, 
+            tensor_scatter_nd_update,
+            sin, 
+            sin_, 
+            sinh, 
+            sign, 
+            selu, 
+            silu, 
+            slice, 
+            slice_update,
+            softsign, 
+            sort, 
+            softplus, 
+            sigmoid, 
+            softmax, 
+            squeeze, 
+            split, 
+            stack, 
+            std,
+            sub, 
+            sum, 
+            sqrt, 
+            square,  
+            swapaxes, 
+            swapdims, 
+            tan, 
+            tanh, 
+            tensor, 
+            tensordot,
+            tile, 
+            transpose,
+            t,
+            tril, 
+            unsqueeze, 
+            unbind, 
+            permute,
+            var, 
+            where, 
+            zeros, 
+            zeros_like,
+            is_nonzero,
+            is_tensor,
+            no_grad,
+            set_grad_enabled,
+            enable_grad,
+            inference_mode,
+            is_grad_enabled,
+            is_floating_point,
+            set_printoptions,
+            decode_onerec,
+            from_numpy,
+            as_tensor,
+            cumsum,
+            topk,
+            nms,
+            cumprod,
+            HalfTensor,
+            FloatTensor,
+            DoubleTensor,
+            BoolTensor,
+            ByteTensor,
+            CharTensor,
+            IntTensor,
+            LongTensor,
+            seed,
+            manual_seed,
+            initial_seed,
+            get_rng_state,
+            set_rng_state,
+            isnan,
+            isinf,
+            searchsorted
+
+.. autofunction:: oneflow.relu
+.. autofunction:: oneflow.set_num_threads

docs/source/optim.rst

+oneflow.optim
+===================================
+Optimizers
+----------------------------------
+.. currentmodule:: oneflow.optim
+.. automodule:: oneflow.optim
+    :members: Adam,
+        Adagrad, 
+        AdamW,
+        Optimizer,
+        RMSprop,
+        SGD,
+        LAMB,
+        lr_scheduler
+
+.. automodule:: oneflow.optim.lr_scheduler
+    :members: CosineDecayLR,
+            CosineAnnealingLR,
+            LambdaLR,
+            StepLR,
+            MultiStepLR,
+            ExponentialLR,
+            ReduceLROnPlateau,
+            PolynomialLR

docs/source/tensor.rst

+oneflow.Tensor
+===================================
+OneFlow Tensor Class
+----------------------------------
+.. currentmodule:: oneflow
+.. autoclass:: oneflow.Tensor
+    :members: abs, 
+            acos, 
+            acosh, 
+            add, 
+            add_, 
+            addcmul,
+            addcmul_,
+            addmm,
+            amin,
+            amax,
+            arccos, 
+            arccosh, 
+            arcsin, 
+            arcsinh, 
+            arctan, 
+            arctanh, 
+            argmax, 
+            argmin, 
+            argsort, 
+            argwhere, 
+            asin, 
+            asinh, 
+            atan, 
+            atan2, 
+            atanh, 
+            backward,
+            bmm, 
+            byte, 
+            cast, 
+            ceil, 
+            chunk,  
+            clamp, 
+            clamp_,
+            clip, 
+            clip_, 
+            clone, 
+            copy_, 
+            cos, 
+            cosh, 
+            cpu, 
+            cuda,
+            data, 
+            dot,
+            detach, 
+            device, 
+            placement,
+            sbp,
+            diag, 
+            diagonal,
+            dim, 
+            div, 
+            div_, 
+            double, 
+            dtype, 
+            element_size, 
+            eq, 
+            erf, 
+            erfc, 
+            erfinv, 
+            erfinv_, 
+            exp, 
+            expand, 
+            expand_as, 
+            expm1, 
+            fill_, 
+            flatten, 
+            flip, 
+            float, 
+            floor, 
+            floor_, 
+            fmod,
+            gather, 
+            ge, 
+            gelu, 
+            get_device, 
+            grad, 
+            grad_fn, 
+            gt, 
+            half,
+            in_top_k, 
+            index_select,
+            int, 
+            is_global, 
+            is_contiguous, 
+            is_cuda, 
+            is_floating_point, 
+            is_lazy, 
+            is_leaf, 
+            item, 
+            le, 
+            log, 
+            log1p,
+            logical_and,
+            logical_or,
+            logical_not,
+            logical_xor,
+            long, 
+            lt, 
+            masked_fill, 
+            masked_select, 
+            matmul, 
+            mm, 
+            mv, 
+            max, 
+            mean, 
+            min, 
+            mish, 
+            mul, 
+            mul_, 
+            narrow, 
+            ndim, 
+            ndimension, 
+            ne, 
+            negative, 
+            nelement, 
+            new_empty,
+            new_ones, 
+            new_zeros,
+            nonzero,
+            norm, 
+            normal_, 
+            numel, 
+            numpy, 
+            permute, 
+            pow, 
+            prod,
+            reciprocal, 
+            register_hook, 
+            relu,
+            repeat,
+            repeat_interleave,
+            requires_grad,
+            requires_grad_,
+            reshape, 
+            retain_grad,
+            roll,
+            round, 
+            rsqrt, 
+            selu, 
+            shape, 
+            sigmoid, 
+            sign, 
+            silu, 
+            sin, 
+            sin_, 
+            sinh, 
+            size, 
+            softmax, 
+            softplus, 
+            softsign, 
+            sort, 
+            split, 
+            sqrt, 
+            square, 
+            squeeze, 
+            std, 
+            storage_offset, 
+            stride, 
+            sum,
+            swapaxes, 
+            swapdims, 
+            sub, 
+            sub_, 
+            tan, 
+            tanh, 
+            tile, 
+            to,
+            local_to_global,
+            global_to_global,
+            to_global,
+            to_local,
+            to_consistent,
+            tolist, 
+            topk, 
+            transpose,
+            tril, 
+            triu, 
+            type_as, 
+            type,
+            t,
+            T,
+            unbind, 
+            unfold, 
+            uniform_, 
+            unsqueeze, 
+            var, 
+            view, 
+            view_as, 
+            where, 
+            zero_, 
+            nms,
+            pin_memory,
+            is_pinned,
+

docs/source/tensor_attributes.rst

+.. currentmodule:: oneflow
+
+Tensor Attributes
+=============================================================
+Each local ``oneflow.Tensor`` has a :class:`oneflow.dtype`, :class:`oneflow.device`, and global ``oneflow.Tensor`` has a :class:`oneflow.dtype`, :class:`oneflow.placement`, :class:`oneflow.sbp`.
+
+oneflow.device
+--------------------------------------------------------------
+.. autoclass:: oneflow.device
+
+oneflow.placement
+--------------------------------------------------------------
+.. autoclass:: oneflow.placement
+
+oneflow.env.all_device_placement
+--------------------------------------------------------------
+.. autofunction:: oneflow.env.all_device_placement
+
+oneflow.sbp.sbp
+--------------------------------------------------------------
+.. autoclass:: oneflow.sbp.sbp

docs/source/troubleshooting.md

+# Troubleshooting
+
+- 'libunwind.h' not found
+
+  - You might add CMake argument `-DWITH_UNWIND=OFF`, or install libunwind in your system.
+
+- `CUDNN_STATUS_NOT_INITIALIZED`
+
+  - You might see error message like these:
+    ```
+    I0729 22:37:45.483937439   56788 ev_epoll_linux.c:82]        Use of signals is disabled. Epoll enginll not be used
+    E0729 22:37:45.515343 56788 version.cpp:82] Failed to get cuda runtime version: CUDA driver version nsufficient for CUDA runtime version
+    F0729 22:38:31.209002 56788 improver.cpp:535] Check failed: mem_size > 0 (-524288000 vs. 0)
+    ```
+    ```
+    F0723 19:05:56.194067 40970 cuda_util.cpp:82] Check failed: error == CUDNN_STATUS_SUCCESS (1 vs. 0) CUDNN_STATUS_NOT_INITIALIZED
+    ```
+  - Please upgrade to Nvidia Linux x86_64 driver. Version >= 440.33 is recommended.
+  - For more information, please refer to [CUDA compatibility documentation](https://docs.nvidia.com/deploy/cuda-compatibility/index.html).
+
+- Failed to compile `.cu` files
+
+  - Please refer to [CUDA System Requirements](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#system-requirements) . Make sure your linux distribution and libraries shipped with it meet the requirements.
+  - If you are using tools like conda, please make sure libraries you install doesn't shade the proper installation comes with linux distribution or package management like apt-get.
+  - Please build OneFlow with a newer version of CMake. You could download version 3.14 from here: [https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz](https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz)
+
+- How do I know what compilers and flags are used to compile OneFlow?
+
+  - run `make clean && make VERBOSE=1` to get exact compile commands with compiler path and flags
+
+- How to compile OneFlow with RDMA support?
+
+  - add cmake flag `-DBUILD_RDMA` to compile OneFlow
+
+- Which version of g++ CMake is using to build OneFlow?
+
+  - You should find a line like this in CMake output:
+
+    ```bash
+    -- CMAKE_CXX_COMPILER_VERSION: [YOUR G++ VERSION NUMBER]
+    ```
+
+- Failed to compile NCCL
+
+  - Try use less threads when compiling OneFlow third party. For instance, use
+
+    ```bash
+    cmake -DTHIRD_PARTY=ON .. && make
+    ```
+
+    instead of
+
+    ```bash
+    cmake -DTHIRD_PARTY=ON .. && make -j$(nproc) `
+    ```
+
+- `"CUDA_VERSION" "VERSION_GREATER_EQUAL" "10.0"`
+
+  - Please use a newer version of CMake
+  - Make sure cmake is correctly included in `PATH`
+
+- CUBLAS not found
+
+  - Usually it happens when using CUDA 10.1 or newer
+  - You should see error massage by CMake like this:
+
+    ```
+    cuda lib not found: /usr/local/miniconda3/envs/dl/lib/libcublas_static.a or
+    /usr/local/cuda/lib64/libcublas_static.a
+    ```
+
+  - Make sure `libcublas_static.a` is in one of the two directories.
+
+- When running OneFlow in gdb, there is no debug information for code location.
+
+  - add cmake flag `-DCMAKE_BUILD_TYPE=RELWITHDEBINFO` or `-DCMAKE_BUILD_TYPE=DEBUG` and recompile
+
+- `libof_ccobj.a: File truncated`
+
+  - You might see error message like this:
+
+    ```
+    /usr/bin/ar: libof_ccobj.a: File truncated
+    make[2]: *** [libof_ccobj.a] Error 1
+    make[2]: *** Deleting file `libof_ccobj.a'
+    make[1]: *** [CMakeFiles/of_ccobj.dir/all] Error 2
+    make: *** [all] Error 2
+    ```
+
+  - You should upgrade your GNU Binutils. Version 2.33.1 is recommended. If you are using conda, you could install it by running `conda install -c conda-forge binutils`
+
+- Failed to compile because C++ 17 is enabled
+
+  - In some cases, environment variable `CXXFLAGS` is not empty and contains `--std c++17`.
+  - Check if it is empty by running `echo $CXXFLAGS` and clear it with `unset CXXFLAGS`.
+  - If you are using conda, to make the changes on environment variables permanent, you can run:
+    ```bash
+    conda env config vars set CXXFLAGS="-fPIC"
+    ```
+
+- cmake outputs error `No CMAKE_ASM_NASM_COMPILER could be found.`
+
+  - Install `nasm`. For instance, run `sudo yum install nasm` if you are on centos.
+
+- `No module named 'google.protobuf'`
+
+  - You might see error message like this:
+    ```
+    Scanning dependencies of target generate_api
+    ...
+        from google.protobuf import descriptor as _descriptor
+    ModuleNotFoundError: No module named 'google.protobuf'
+    CMakeFiles/generate_api.dir/build.make:57: recipe for target 'CMakeFiles/generate_api' failed
+    make[2]: *** [CMakeFiles/generate_api] Error 1
+    ```
+  - Install development dependencies by running:
+    ```
+    pip3 install -r dev-requirements.txt
+    ```
+
+- Get gdb warning `ptrace: Operation not permitted.` and gdb command `bt` prints no backtrace
+
+  - You might get this warning when debugging OneFlow with gdb inside a docker container. Try add these flags when launching your container:
+    ```
+    docker run --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
+    ```
+  - Please refer to https://stackoverflow.com/questions/19215177/how-to-solve-ptrace-operation-not-permitted-when-trying-to-attach-gdb-to-a-pro
+
+- It takes too long to download python packages when running `make`
+  - If you are in China, you could run this to have pip download packages from domestic mirror of pypi:
+    ```
+    python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
+    ```
+  - For more information on this, please refer to [pypi 镜像使用帮助](https://mirror.tuna.tsinghua.edu.cn/help/pypi/)

docs/source/utils.rst

+oneflow.utils
+===================================
+Utils
+----------------------------------
+.. currentmodule:: oneflow.utils
+.. automodule:: oneflow.utils.data
+    :members: DataLoader,
+        Dataset,
+        IterableDataset,
+        TensorDataset,
+        ConcatDataset,
+        Subset,
+        random_split,
+        Sampler,
+        SequentialSampler,
+        RandomSampler,
+        SubsetRandomSampler,
+        BatchSampler
+        
+
+.. currentmodule:: oneflow.utils
+.. automodule:: oneflow.utils.data.distributed
+    :members: DistributedSampler
+
+
+.. autofunction:: oneflow.utils.from_torch
+.. autofunction:: oneflow.utils.to_torch

external/CMakeLists.txt

+set(EXTERNAL_TARGETS)
+
+if(CPU_THREADING_RUNTIME STREQUAL "TBB")
+  add_subdirectory(onetbb)
+  list(APPEND EXTERNAL_TARGETS tbb)
+endif()
+
+add_subdirectory(robin-hood-hashing)
+list(APPEND EXTERNAL_TARGETS robin_hood)
+
+add_subdirectory(fmt)
+list(APPEND EXTERNAL_TARGETS fmt)
+
+add_subdirectory(kineto)
+list(APPEND EXTERNAL_TARGETS kineto)
+
+mark_targets_as_system(${EXTERNAL_TARGETS})
+set_property(GLOBAL PROPERTY EXTERNAL_TARGETS ${EXTERNAL_TARGETS})

external/fmt/CMakeLists.txt

+include(FetchContent)
+
+set(FMT_INSTALL_DIR ${THIRD_PARTY_DIR}/fmt)
+
+FetchContent_Declare(fmt URL ${FMT_URL} URL_HASH MD5=${FMT_MD5})
+
+FetchContent_MakeAvailable(fmt)
+
+install(
+  TARGETS fmt
+  EXPORT oneflow
+  LIBRARY DESTINATION ${FMT_INSTALL_DIR}/lib
+  ARCHIVE DESTINATION ${FMT_INSTALL_DIR}/lib)
+install(DIRECTORY ${fmt_SOURCE_DIR}/include DESTINATION ${FMT_INSTALL_DIR})

external/kineto/CMakeLists.txt

+include(FetchContent)
+
+# reference: https://github.com/PaddlePaddle/Paddle/blob/develop/cmake/cupti.cmake
+
+set(CUPTI_ROOT "/usr" CACHE PATH "CUPTI ROOT")
+
+set(CUDA_SOURCE_DIR ${CUDAToolkit_TARGET_DIR})
+
+find_path(
+  CUPTI_INCLUDE_DIR cupti.h
+  PATHS ${CUPTI_ROOT}
+        ${CUPTI_ROOT}/include
+        $ENV{CUPTI_ROOT}
+        $ENV{CUPTI_ROOT}/include
+        ${CUDA_SOURCE_DIR}/extras/CUPTI/include
+        ${CUDA_SOURCE_DIR}/targets/x86_64-linux/include
+        ${CUDA_SOURCE_DIR}/targets/aarch64-linux/include
+  NO_DEFAULT_PATH)
+
+set(TARGET_ARCH "x86_64")
+if(NOT ${CMAKE_SYSTEM_PROCESSOR})
+  set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR})
+endif()
+
+list(
+  APPEND
+  CUPTI_CHECK_LIBRARY_DIRS
+  ${CUPTI_ROOT}
+  ${CUPTI_ROOT}/lib64
+  ${CUPTI_ROOT}/lib
+  ${CUPTI_ROOT}/lib/${TARGET_ARCH}-linux-gnu
+  $ENV{CUPTI_ROOT}
+  $ENV{CUPTI_ROOT}/lib64
+  $ENV{CUPTI_ROOT}/lib
+  /usr/lib
+  ${CUDA_SOURCE_DIR}/targets/x86_64-linux/lib64
+  ${CUDA_SOURCE_DIR}/extras/CUPTI/lib64)
+
+find_library(
+  CUDA_cupti_LIBRARY
+  NAMES libcupti.so libcupti.dylib # libcupti_static.a
+  PATHS ${CUPTI_CHECK_LIBRARY_DIRS} ${CUPTI_INCLUDE_DIR}
+  NO_DEFAULT_PATH
+  DOC "Path to cuPTI library.")
+
+list(APPEND CUDA_cupti_LIBRARY CUDA::cudart_static) # for undefined symbol: cudaGetDeviceCount∂
+
+FetchContent_Declare(
+  kineto
+  URL ${KINETO_URL}
+  URL_HASH MD5=${KINETO_MD5}
+  SOURCE_SUBDIR libkineto)
+
+FetchContent_MakeAvailable(kineto)
+
+target_include_directories(kineto PUBLIC $<BUILD_INTERFACE:${kineto_SOURCE_DIR}/libkineto/include>)

external/onetbb/CMakeLists.txt

+find_package(Threads REQUIRED)
+set(ONETBB_INSTALL_DIR ${THIRD_PARTY_DIR}/tbb CACHE PATH " ")
+
+include(FetchContent)
+FetchContent_Declare(
+  tbb
+  URL ${ONETBB_URL}
+  URL_HASH MD5=${ONETBB_MD5}
+)
+FetchContent_GetProperties(tbb)
+
+set(TBB_EXAMPLES OFF CACHE BOOL "")
+set(TBB_TEST OFF CACHE BOOL "")
+set(BUILD_SHARED_LIBS ON)
+set(CMAKE_POLICY_DEFAULT_CMP0079 NEW)
+
+FetchContent_MakeAvailable(tbb)
+
+install(TARGETS tbb tbbmalloc tbbmalloc_proxy COMPONENT OneFlowTBB)
+install(DIRECTORY ${tbb_SOURCE_DIR}/include DESTINATION ${ONETBB_INSTALL_DIR} COMPONENT OneFlowTBB)
+
+add_custom_target(install-tbb
+  DEPENDS tbb tbbmalloc tbbmalloc_proxy
+  COMMAND
+      "${CMAKE_COMMAND}" -DCMAKE_INSTALL_PREFIX=${ONETBB_INSTALL_DIR}
+      -DCMAKE_INSTALL_COMPONENT=OneFlowTBB
+      -P "${CMAKE_BINARY_DIR}/cmake_install.cmake"
+)

external/robin-hood-hashing/CMakeLists.txt

+include(FetchContent)
+FetchContent_Declare(
+        robin_hood_hashing
+        URL ${ROBIN_HOOD_HASHING_URL}
+        URL_HASH MD5=${ROBIN_HOOD_HASHING_MD5}
+)
+FetchContent_MakeAvailable(robin_hood_hashing)