Oneflow 0.8 for DCU

cmake/third_party/zlib.cmake

+include(ExternalProject)
+
+set(ZLIB_INSTALL ${THIRD_PARTY_DIR}/zlib)
+set(ZLIB_INCLUDE_DIR ${ZLIB_INSTALL}/include)
+set(ZLIB_LIBRARY_DIR ${ZLIB_INSTALL}/lib)
+set(ZLIB_URL https://github.com/madler/zlib/archive/v1.2.8.tar.gz)
+use_mirror(VARIABLE ZLIB_URL URL ${ZLIB_URL})
+
+# only use zlib shared lib to prevent using zlib in the system
+if(WIN32)
+  set(ZLIB_LIBRARY_NAMES zlibstaticd.lib)
+else()
+  if("${CMAKE_SHARED_LIBRARY_SUFFIX}" STREQUAL ".dylib")
+    set(ZLIB_LIBRARY_NAMES libz.dylib)
+  elseif("${CMAKE_SHARED_LIBRARY_SUFFIX}" STREQUAL ".so")
+    set(ZLIB_LIBRARY_NAMES libz.so)
+  else()
+    message(FATAL_ERROR "${CMAKE_SHARED_LIBRARY_SUFFIX} not support for zlib")
+  endif()
+endif()
+
+foreach(LIBRARY_NAME ${ZLIB_LIBRARY_NAMES})
+  list(APPEND ZLIB_STATIC_LIBRARIES ${ZLIB_LIBRARY_DIR}/${LIBRARY_NAME})
+endforeach()
+
+set(ZLIB_HEADERS "${ZLIB_INSTALL}/include/zconf.h" "${ZLIB_INSTALL}/include/zlib.h")
+
+if(THIRD_PARTY)
+
+  ExternalProject_Add(
+    zlib
+    PREFIX zlib
+    URL ${ZLIB_URL}
+    URL_MD5 1eabf2698dc49f925ce0ffb81397098f
+    UPDATE_COMMAND ""
+    BUILD_IN_SOURCE 1
+    BUILD_BYPRODUCTS ${ZLIB_STATIC_LIBRARIES}
+    CMAKE_CACHE_ARGS
+      -DCMAKE_C_COMPILER_LAUNCHER:STRING=${CMAKE_C_COMPILER_LAUNCHER}
+      -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=${CMAKE_CXX_COMPILER_LAUNCHER}
+      -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
+      -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS}
+      -DCMAKE_CXX_FLAGS:STRING=${CMAKE_CXX_FLAGS}
+      -DCMAKE_CXX_FLAGS_DEBUG:STRING=${CMAKE_CXX_FLAGS_DEBUG}
+      -DCMAKE_CXX_FLAGS_RELEASE:STRING=${CMAKE_CXX_FLAGS_RELEASE}
+      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+      -DCMAKE_INSTALL_PREFIX:STRING=${ZLIB_INSTALL}
+      -DCMAKE_INSTALL_MESSAGE:STRING=${CMAKE_INSTALL_MESSAGE})
+
+endif(THIRD_PARTY)
+add_library(zlib_imported UNKNOWN IMPORTED)
+set_property(TARGET zlib_imported PROPERTY IMPORTED_LOCATION "${ZLIB_STATIC_LIBRARIES}")

cmake/util.cmake

+function(SHOW_VARIABLES)
+  get_cmake_property(_variableNames VARIABLES)
+  foreach(_variableName ${_variableNames})
+    message(STATUS "${_variableName}=${${_variableName}}")
+  endforeach()
+endfunction()
+
+macro(write_file_if_different file_path content)
+  if(EXISTS ${file_path})
+    file(READ ${file_path} current_content)
+    # NOTE: it seems a cmake bug that "content" in this macro is not
+    # treated as a variable
+    if(NOT (current_content STREQUAL ${content}))
+      file(WRITE ${file_path} ${content})
+    endif()
+  else()
+    file(WRITE ${file_path} ${content})
+  endif()
+endmacro()
+
+macro(copy_all_files_in_dir source_dir dest_dir target)
+  find_program(rsync rsync)
+  if(rsync)
+    add_custom_command(
+      TARGET ${target}
+      POST_BUILD
+      COMMAND
+        ${rsync}
+        # NOTE: the trailing slash of source_dir is needed.
+        # Reference: https://stackoverflow.com/a/56627246
+        ARGS -a --omit-dir-times --no-perms --no-owner --no-group --inplace ${source_dir}/
+        ${dest_dir})
+  else()
+    add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory
+                                                           ${source_dir} ${dest_dir})
+  endif()
+endmacro()
+
+set(_COUNTER 0)
+macro(copy_files file_paths source_dir dest_dir target)
+  find_program(rsync rsync)
+  if(rsync)
+    set(CACHE_FILELIST ${PROJECT_BINARY_DIR}/cached_filename_lists/cache_${_COUNTER})
+    math(EXPR _COUNTER "${_COUNTER} + 1")
+    file(WRITE ${CACHE_FILELIST} "")
+    foreach(file ${file_paths})
+      file(RELATIVE_PATH rel_path "${source_dir}" ${file})
+      file(APPEND ${CACHE_FILELIST} ${rel_path}\n)
+    endforeach()
+    add_custom_command(
+      TARGET ${target} POST_BUILD
+      COMMAND ${rsync} ARGS -a --omit-dir-times --no-perms --no-owner --no-group --inplace
+              --files-from=${CACHE_FILELIST} ${source_dir} ${dest_dir})
+  else()
+    foreach(file ${file_paths})
+      file(RELATIVE_PATH rel_path "${source_dir}" ${file})
+      add_custom_command(TARGET ${target} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                                                             "${file}" "${dest_dir}/${rel_path}")
+    endforeach()
+  endif()
+endmacro()
+
+function(add_copy_headers_target)
+  cmake_parse_arguments(PARSED_ARGS "" "NAME;SRC;DST;INDEX_FILE" "DEPS" ${ARGN})
+  if(NOT PARSED_ARGS_NAME)
+    message(FATAL_ERROR "name required")
+  endif(NOT PARSED_ARGS_NAME)
+  if(NOT PARSED_ARGS_SRC)
+    message(FATAL_ERROR "src required")
+  endif(NOT PARSED_ARGS_SRC)
+  if(NOT PARSED_ARGS_DST)
+    message(FATAL_ERROR "dst required")
+  endif(NOT PARSED_ARGS_DST)
+  add_custom_target(
+    "${PARSED_ARGS_NAME}_create_header_dir" COMMAND ${CMAKE_COMMAND} -E make_directory
+                                                    "${PARSED_ARGS_DST}"
+    DEPENDS ${PARSED_ARGS_DEPS})
+
+  add_custom_target("${PARSED_ARGS_NAME}_copy_headers_to_destination" ALL
+                    DEPENDS "${PARSED_ARGS_NAME}_create_header_dir")
+  file(GLOB_RECURSE headers "${PARSED_ARGS_SRC}/*.h")
+  file(GLOB_RECURSE cuda_headers "${PARSED_ARGS_SRC}/*.cuh")
+  file(GLOB_RECURSE hpp_headers "${PARSED_ARGS_SRC}/*.hpp")
+  list(APPEND headers ${cuda_headers})
+  list(APPEND headers ${hpp_headers})
+
+  foreach(header_file ${headers})
+    file(RELATIVE_PATH relative_file_path ${PARSED_ARGS_SRC} ${header_file})
+    add_custom_command(
+      TARGET "${PARSED_ARGS_NAME}_copy_headers_to_destination" PRE_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${header_file}
+              "${PARSED_ARGS_DST}/${relative_file_path}")
+  endforeach()
+
+  if(PARSED_ARGS_INDEX_FILE)
+    file(STRINGS ${PARSED_ARGS_INDEX_FILE} inventory_headers)
+  endif(PARSED_ARGS_INDEX_FILE)
+  foreach(header_file ${inventory_headers})
+    add_custom_command(
+      TARGET "${PARSED_ARGS_NAME}_copy_headers_to_destination" PRE_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different "${PARSED_ARGS_SRC}/${header_file}"
+              "${PARSED_ARGS_DST}/${header_file}")
+  endforeach()
+endfunction()
+
+function(use_mirror)
+  set(ALIYUN_URL_PREFIX
+      "https://oneflow-static.oss-cn-beijing.aliyuncs.com/third_party_mirror/https/"
+      CACHE STRING "URL prefix of Aliyun OSS mirror")
+  cmake_parse_arguments(PARSED_ARGS "" "VARIABLE;URL" "" ${ARGN})
+
+  if((NOT PARSED_ARGS_VARIABLE) OR (NOT PARSED_ARGS_URL))
+    message(FATAL_ERROR "VARIABLE or URL required")
+  endif()
+
+  if(PARSED_ARGS_URL MATCHES "file://")
+    set(${PARSED_ARGS_VARIABLE} ${PARSED_ARGS_URL} PARENT_SCOPE)
+    return()
+  endif()
+  if(DEFINED THIRD_PARTY_MIRROR)
+    if(THIRD_PARTY_MIRROR STREQUAL "aliyun")
+      if(NOT PARSED_ARGS_URL MATCHES "^https://")
+        message(FATAL_ERROR "URL should start with 'https://'")
+      endif()
+      string(REPLACE "https://" ${ALIYUN_URL_PREFIX} MIRRORED_URL ${PARSED_ARGS_URL})
+      set(${PARSED_ARGS_VARIABLE} ${MIRRORED_URL} PARENT_SCOPE)
+      message(NOTICE "-- fetch ${PARSED_ARGS_VARIABLE} using aliyun mirror ${MIRRORED_URL}")
+    elseif(NOT THIRD_PARTY_MIRROR STREQUAL "")
+      message(FATAL_ERROR "invalid key for third party mirror")
+    endif()
+  endif()
+endfunction()
+
+macro(set_mirror_url variable url)
+  set(${variable} ${url} ${ARGN})
+  use_mirror(VARIABLE ${variable} URL ${url})
+endmacro()
+
+macro(set_mirror_url_with_hash variable url hash)
+  set_mirror_url(${variable} ${url} ${ARGN})
+  set(${variable}_HASH ${hash} ${ARGN})
+endmacro()
+
+function(check_cxx11_abi OUTPUT_VAR)
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} -E echo "#include <string>\n void test(std::string){}\n int main(){}"
+    OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/temp.cpp)
+  try_compile(
+    COMPILE_SUCCESS ${CMAKE_CURRENT_BINARY_DIR}
+    ${CMAKE_CURRENT_BINARY_DIR}/temp.cpp
+    COMPILE_DEFINITIONS -D_GLIBCXX_USE_CXX11_ABI=1
+    COPY_FILE ${CMAKE_CURRENT_BINARY_DIR}/temp)
+  if(NOT COMPILE_SUCCESS)
+    message(FATAL_ERROR "Detecting cxx11 availability failed. Please report to OneFlow developers.")
+  endif()
+  execute_process(COMMAND nm ${CMAKE_CURRENT_BINARY_DIR}/temp COMMAND grep -q cxx11
+                  RESULT_VARIABLE RET_CODE)
+  if(RET_CODE EQUAL 0)
+    set(CXX11_ABI_AVAILABLE ON)
+  else()
+    set(CXX11_ABI_AVAILABLE OFF)
+  endif()
+  execute_process(COMMAND rm ${CMAKE_CURRENT_BINARY_DIR}/temp ${CMAKE_CURRENT_BINARY_DIR}/temp.cpp)
+  set(${OUTPUT_VAR} ${CXX11_ABI_AVAILABLE} PARENT_SCOPE)
+endfunction()
+
+include(CheckCXXCompilerFlag)
+
+function(target_try_compile_option target flag)
+  # We cannot check for -Wno-foo as this won't throw a warning so we must check for the -Wfoo option directly
+  # http://stackoverflow.com/questions/38785168/cc1plus-unrecognized-command-line-option-warning-on-any-other-warning
+  string(REGEX REPLACE "^-Wno-" "-W" checkedFlag ${flag})
+  string(REGEX REPLACE "[-=]" "_" varName CXX_FLAG${checkedFlag})
+  # Avoid double checks. A compiler will not magically support a flag it did not before
+  if(NOT DEFINED ${varName}_SUPPORTED)
+    check_cxx_compiler_flag(${checkedFlag} ${varName}_SUPPORTED)
+  endif()
+  if(${varName}_SUPPORTED)
+    target_compile_options(${target} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:${flag}>)
+    if(BUILD_CUDA)
+      if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND "${CMAKE_CUDA_COMPILER_ID}" STREQUAL
+                                                         "Clang")
+        target_compile_options(${target} PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:${flag}>)
+      endif()
+    endif()
+  endif()
+endfunction()
+
+function(target_try_compile_options target)
+  foreach(flag ${ARGN})
+    target_try_compile_option(${target} ${flag})
+  endforeach()
+endfunction()
+
+function(target_treat_warnings_as_errors target)
+  if(TREAT_WARNINGS_AS_ERRORS)
+    target_compile_options(${target} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:-Werror>)
+    if(BUILD_CUDA)
+      # Only pass flags when cuda compiler is Clang because cmake handles -Xcompiler incorrectly
+      if("${CMAKE_CUDA_COMPILER_ID}" STREQUAL "Clang")
+        target_compile_options(${target} PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Werror>)
+      endif()
+    endif()
+
+    # TODO: remove it while fixing all deprecated call
+    target_try_compile_options(${target} -Wno-error=deprecated-declarations)
+
+    # disable unused-* for different compile mode (maybe unused in cpu.cmake, but used in cuda.cmake)
+    target_try_compile_options(
+      ${target} -Wno-error=unused-const-variable -Wno-error=unused-variable
+      -Wno-error=unused-local-typedefs -Wno-error=unused-private-field
+      -Wno-error=unused-lambda-capture)
+
+    # there is some strict-overflow warnings in oneflow/user/kernels/ctc_loss_kernel_util.cpp for unknown reason, disable them for now
+    target_try_compile_options(${target} -Wno-error=strict-overflow)
+
+    target_try_compile_options(${target} -Wno-error=instantiation-after-specialization)
+
+    # disable for pointer operations of intrusive linked lists
+    target_try_compile_options(${target} -Wno-error=array-bounds)
+
+    target_try_compile_options(${target} -Wno-error=comment)
+
+    # disable visibility warnings related to https://github.com/Oneflow-Inc/oneflow/pull/3676.
+    target_try_compile_options(${target} -Wno-error=attributes)
+  endif()
+endfunction()
+
+function(set_compile_options_to_oneflow_target target)
+  target_treat_warnings_as_errors(${target})
+  target_compile_options(${target} PRIVATE $<$<COMPILE_LANGUAGE:CXX>:-Werror=return-type>)
+  # the mangled name between `struct X` and `class X` is different in MSVC ABI, remove it while windows is supported (in MSVC/cl or clang-cl)
+  target_try_compile_options(${target} -Wno-covered-switch-default)
+
+  if(OMP_FLAGS)
+    target_try_compile_options(${target} ${OMP_FLAGS})
+  endif()
+
+  set_target_properties(${target} PROPERTIES INSTALL_RPATH "$ORIGIN/../lib")
+
+  if(BUILD_CUDA)
+    if("${CMAKE_CUDA_COMPILER_ID}" STREQUAL "NVIDIA")
+      target_compile_options(
+        ${target}
+        PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:
+                -Xcompiler
+                -Werror=return-type;
+                -Wno-deprecated-gpu-targets;
+                -Werror
+                cross-execution-space-call;
+                -Xcudafe
+                --diag_suppress=declared_but_not_referenced;
+                >)
+    elseif("${CMAKE_CUDA_COMPILER_ID}" STREQUAL "Clang")
+      target_compile_options(
+        ${target}
+        PRIVATE
+          $<$<COMPILE_LANGUAGE:CUDA>:
+          -Werror=return-type;
+          # Suppress warning from cub library -- marking as system header seems not working for .cuh files
+          -Wno-pass-failed;
+          >)
+    else()
+      message(FATAL_ERROR "Unknown CUDA compiler ${CMAKE_CUDA_COMPILER_ID}")
+    endif()
+    # remove THRUST_IGNORE_CUB_VERSION_CHECK if starting using bundled cub
+    target_compile_definitions(${target} PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:
+                                                 THRUST_IGNORE_CUB_VERSION_CHECK; >)
+  endif()
+endfunction()
+
+function(check_variable_defined variable)
+  if(NOT DEFINED ${variable})
+    message(FATAL_ERROR "Variable ${variable} is not defined")
+  endif()
+endfunction()
+
+function(checkDirAndAppendSlash)
+  set(singleValues DIR;OUTPUT)
+  set(prefix ARG)
+  cmake_parse_arguments(PARSE_ARGV 0 ${prefix} "${noValues}" "${singleValues}" "${multiValues}")
+
+  if("${${prefix}_DIR}" STREQUAL "" OR "${${prefix}_DIR}" STREQUAL "/")
+    message(FATAL_ERROR "emtpy path found: ${${prefix}_DIR}")
+  else()
+    set(${${prefix}_OUTPUT} "${${prefix}_DIR}/" PARENT_SCOPE)
+  endif()
+
+endfunction()
+
+function(mark_targets_as_system)
+  # TODO(daquexian): update this function once https://gitlab.kitware.com/cmake/cmake/-/merge_requests/7308
+  # and its following PRs are merged in cmake v3.25.
+  foreach(target ${ARGV})
+    get_target_property(include_dir ${target} INTERFACE_INCLUDE_DIRECTORIES)
+    set_target_properties(${target} PROPERTIES INTERFACE_SYSTEM_INCLUDE_DIRECTORIES
+                                               "${include_dir}")
+  endforeach()
+endfunction()

dev-requirements.txt

+black==19.10b0; python_version >= "3.6"
+click==8.0.0; python_version >= "3.6" # https://github.com/psf/black/issues/2964
+numpy>=1.17.0
+protobuf>=3.9.2, <4.0
+wheel
+tqdm
+requests
+jinja2
+opencv-python; python_version >= "3.9" and sys_platform != 'darwin' and platform_machine != 'aarch64'
+opencv-python==4.2.0.34; python_version < '3.9' and sys_platform != 'darwin' and platform_machine != 'aarch64'
+PyYAML>=5.1
+pillow
+dataclasses; python_version<"3.7"
+cmakelang==0.6.13
+pytest-xdist
+rich

docker/build/Dockerfile

+# warning: never share the container image this dockerfile produces
+ARG CUDA=10.0
+
+FROM nvidia/cuda:${CUDA}-cudnn7-devel-centos7
+RUN yum-config-manager --add-repo https://yum.repos.intel.com/setup/intelproducts.repo && \
+    rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+RUN yum update -y && yum install -y epel-release
+RUN yum update -y && yum install -y rdma-core-devel \
+    nasm \
+    cmake3 \
+    make \
+    git \
+    centos-release-scl \
+    intel-mkl-2020.0-088 \
+    zlib-devel \
+    curl-devel \
+    which
+
+RUN ln -sf /usr/bin/cmake3 /usr/bin/cmake
+
+RUN mkdir -p /tmp/download/cmake-extracted && \
+    cd /tmp/download && \
+    curl --location https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0.tar.gz --output cmake.tar.gz && \
+    tar -xvzf cmake.tar.gz --directory cmake-extracted && \
+    cd cmake-extracted/* && \
+    mkdir /cmake-install
+RUN cd /tmp/download/cmake-extracted/* && \
+    cmake . -DCMAKE_USE_SYSTEM_CURL=ON -DCMAKE_INSTALL_PREFIX=/cmake-install && \
+    make -j $(nproc) && \
+    make install
+ENV PATH="/cmake-install/bin:${PATH}"
+
+ARG USE_PYTHON_3_OR_2=3
+
+RUN if [ "${USE_PYTHON_3_OR_2}" -eq 2 ] ; then yum update -y \
+    && yum install -y python-devel.x86_64 \
+    && curl https://bootstrap.pypa.io/get-pip.py --output ./get-pip.py \
+    && python ./get-pip.py \
+    && rm get-pip.py \
+    && pip install numpy==1.12.0 protobuf ; fi
+
+COPY dev-requirements.txt /workspace/dev-requirements.txt
+
+RUN if [ "${USE_PYTHON_3_OR_2}" -eq 3 ] ; then yum update -y \
+    && yum install -y rh-python36 python36-devel.x86_64 python36-devel \
+    && python3 -m ensurepip \
+    && pip3 install /workspace/dev-requirements.txt; fi
+
+WORKDIR /workspace/build
+
+COPY cmake /workspace/cmake
+COPY CMakeLists.txt /workspace/CMakeLists.txt
+
+# BUILD DEPENDENCY
+COPY build/third_party /workspace/build/third_party
+RUN cmake -DTHIRD_PARTY=ON -DCMAKE_BUILD_TYPE=Release -DRELEASE_VERSION=ON .. && make -j
+
+# BUILD ONEFLOW
+COPY oneflow /workspace/oneflow
+COPY tools /workspace/tools
+
+RUN export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH; \
+    cmake -DTHIRD_PARTY=OFF .. && make -j $(nproc) ;
+
+## BUILD WHEEL
+WORKDIR /workspace
+RUN pip${USE_PYTHON_3_OR_2} install wheel
+COPY setup.py /workspace/setup.py
+RUN python${USE_PYTHON_3_OR_2} setup.py bdist_wheel
+RUN pip${USE_PYTHON_3_OR_2} install /workspace/dist/*.whl
+
+RUN rm -rf oneflow third_party cmake CMakeLists.txt

docker/build/build-ubuntu.sh

+docker build \
+  --rm \
+  -t oneflow-build:ubuntu -f docker/build/build.ubuntu.dockerfile .

docker/build/build.sh

+docker build \
+  --rm \
+  -t oneflow-build -f docker/build/Dockerfile .

docker/build/build.ubuntu.dockerfile

+ARG CUDA=10.0
+ARG UBUNTU_VERSION=16.04
+FROM nvidia/cuda:${CUDA}-cudnn7-devel-ubuntu${UBUNTU_VERSION}
+
+USER 0
+
+RUN apt-get update && \
+    apt-get install -y apt-transport-https && \
+    apt-get install -y --no-install-recommends \
+    curl \
+    nasm \
+    make \
+    git \
+    gcc \
+    g++ \
+    libopenblas-dev \
+    python3-dev
+
+# speed up pip install in China
+ENV TUNA_PIP_INSTALL=" -i https://pypi.tuna.tsinghua.edu.cn/simple"
+
+COPY dev-requirements.txt /workspace/dev-requirements.txt
+
+RUN curl https://bootstrap.pypa.io/get-pip.py --output ./get-pip.py \
+    && python3 ./get-pip.py \
+    && pip3 install $TUNA_INDEX cmake \
+    && pip3 install $TUNA_INDEX -r /workspace/dev-requirements.txt
+
+WORKDIR /workspace/build
+
+COPY cmake /workspace/cmake
+COPY CMakeLists.txt /workspace/CMakeLists.txt
+
+# BUILD DEPENDENCY
+COPY build/third_party /workspace/build/third_party
+RUN cmake -DTHIRD_PARTY=ON -DONEFLOW=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc)
+
+# BUILD ONEFLOW
+COPY oneflow /workspace/oneflow
+COPY tools /workspace/tools
+
+RUN cmake -DTHIRD_PARTY=OFF -DONEFLOW=ON .. && make -j$(nproc) of_pyscript_copy
+RUN cmake -DTHIRD_PARTY=OFF -DONEFLOW=ON .. && make -j$(nproc)
+
+# BUILD WHEEL
+WORKDIR /workspace
+COPY setup.py /workspace/setup.py
+RUN python3 setup.py bdist_wheel
+RUN pip3 install /workspace/dist/*.whl
+
+RUN rm -rf oneflow third_party cmake CMakeLists.txt

docker/build/launch.sh

+docker run -it --rm \
+	-v /dataset:/dataset/ \
+	oneflow-build 

docker/build/test.sh

+docker run -it --rm \
+	-v /dataset:/dataset/ \
+	oneflow-build \
+    python3 -c "import oneflow"

docker/ci/base/Dockerfile

+# warning: never share the container image this dockerfile produces
+ARG CUDA=10.0
+
+FROM nvidia/cuda:${CUDA}-cudnn7-devel-centos7
+
+COPY dev-requirements.txt /workspace/dev-requirements.txt
+RUN yum-config-manager --add-repo https://yum.repos.intel.com/setup/intelproducts.repo && \
+    rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    yum update -y && yum install -y epel-release && \
+    yum update -y && yum install -y rdma-core-devel \
+    nasm \
+    make \
+    git \
+    centos-release-scl \
+    intel-mkl-2020.0-088 \
+    zlib-devel \
+    curl-devel \
+    which \
+    rh-python36 python36-devel.x86_64 python36-devel && \
+    python3 -m ensurepip && \
+    pip3 install -r /workspace/dev-requirements.txt && \
+    yum clean all
+
+RUN mkdir -p /tmp/download && \
+    mkdir /cmake-extracted && \
+    cd /tmp/download && \
+    curl --location https://github.com/Kitware/CMake/releases/download/v3.14.0/cmake-3.14.0-Linux-x86_64.tar.gz --output cmake.tar.gz && \
+    tar -xvzf cmake.tar.gz --directory /cmake-extracted && \
+    mv /cmake-extracted/* /cmake-extracted/cmake-install && \
+    rm -rf /tmp/download
+
+ENV PATH="/cmake-extracted/cmake-install/bin:${PATH}"

docker/ci/fmt/Dockerfile

+FROM python:3.7
+RUN curl https://oneflow-static.oss-cn-beijing.aliyuncs.com/bin/clang-format -o /usr/local/bin/clang-format && chmod +x /usr/local/bin/clang-format
+RUN apt update && apt install -y libncurses5

docker/ci/fmt/build.sh

+set -ex
+cd docker/ci/fmt
+docker build -t oneflow-fmt .

docker/ci/make/Dockerfile

+ARG from
+FROM ${from}
+WORKDIR /workspace/build
+
+# BUILD ONEFLOW
+COPY oneflow /workspace/oneflow
+COPY tools /workspace/tools
+
+RUN export LD_LIBRARY_PATH=/opt/intel/lib/intel64_lin:/opt/intel/mkl/lib/intel64:$LD_LIBRARY_PATH; \
+    cmake -DTHIRD_PARTY=OFF -DONEFLOW=ON .. && make -j $(nproc) ;
+
+## BUILD WHEEL
+WORKDIR /workspace
+COPY setup.py /workspace/setup.py
+RUN python3 setup.py bdist_wheel
+
+FROM centos:7
+WORKDIR /workspace
+COPY --from=0 /workspace/dist/*.whl .
+COPY --from=0 /workspace/build/bin/oneflow_testexe .

docker/ci/test-v2/Dockerfile

+FROM pytorch/pytorch:1.9.0-cuda10.2-cudnn7-runtime
+COPY sources.list /etc/apt/sources.list
+RUN apt update && apt install ffmpeg libsm6 libxext6 gdb gcc g++ -y --no-install-recommends
+COPY requirements.txt .
+RUN python3 -m pip install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt

docker/ci/test-v2/build.sh

+set -ex
+test_img_dir="$(dirname "${BASH_SOURCE[0]}")"
+test_img_dir="$(realpath "${test_img_dir}")"
+cd $test_img_dir
+
+proxy_args=""
+proxy_args+=" --network=host"
+proxy_args+=" --build-arg HTTP_PROXY=${HTTP_PROXY}"
+proxy_args+=" --build-arg HTTPS_PROXY=${HTTPS_PROXY}"
+proxy_args+=" --build-arg http_proxy=${http_proxy}"
+proxy_args+=" --build-arg https_proxy=${https_proxy}"
+
+img_tag="oneflow-test-v2:0.1" # update me if any of related files are changed
+if [[ "$(docker images -q ${img_tag} 2> /dev/null)" == "" ]]; then
+  docker build --rm $proxy_args \
+    -t $img_tag .
+fi

docker/ci/test-v2/requirements.txt

+sphinx==3.5.4
+jinja2<3.1
+recommonmark==0.6.0
+furo==2021.4.11b34
+sphinx-copybutton==0.5.0
+# dependencies above must be identical to docs/requirements.txt
+pycocotools
+opencv-python==4.2.0.34
+scipy
+pillow
+https://oneflow-static.oss-cn-beijing.aliyuncs.com/pipindex/pipindex-0.1.3-py2.py3-none-any.whl

docker/ci/test-v2/sources.list

+# 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释
+deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse
+# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal main restricted universe multiverse
+deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse
+# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-updates main restricted universe multiverse
+deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse
+# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-backports main restricted universe multiverse
+deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse
+# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-security main restricted universe multiverse
+
+# 预发布软件源，不建议启用
+# deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse
+# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ focal-proposed main restricted universe multiverse

docker/ci/test/Dockerfile

+FROM ufoym/deepo
+
+RUN apt remove openmpi-common libfabric1 openmpi-bin librdmacm1:amd64 libopenmpi2 libopenmpi2:amd64 -y
+ENV MOFED_DIR MLNX_OFED_LINUX-4.3-1.0.1.0-ubuntu18.04-x86_64
+RUN wget https://oneflow-static.oss-cn-beijing.aliyuncs.com/deps/${MOFED_DIR}.tgz && \
+    tar -xzvf ${MOFED_DIR}.tgz && \
+    ${MOFED_DIR}/mlnxofedinstall --user-space-only --without-fw-update --all -q --force && \
+    cd .. && \
+    rm -rf ${MOFED_DIR} && \
+    rm -rf *.tgz
+
+RUN apt update && apt install -y --no-install-recommends gdb openssh-server openssh-client
+
+RUN echo 'ALL ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+COPY requirements.txt .
+RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple -r requirements.txt

docker/ci/test/build.sh

+set -ex
+test_img_dir="$(dirname "${BASH_SOURCE[0]}")"
+test_img_dir="$(realpath "${test_img_dir}")"
+cd $test_img_dir
+
+proxy_args=""
+proxy_args+=" --network=host"
+proxy_args+=" --build-arg HTTP_PROXY=${HTTP_PROXY}"
+proxy_args+=" --build-arg HTTPS_PROXY=${HTTPS_PROXY}"
+proxy_args+=" --build-arg http_proxy=${http_proxy}"
+proxy_args+=" --build-arg https_proxy=${https_proxy}"
+
+img_tag="oneflow-test:0.2" # update me if any of related files are changed
+if [[ "$(docker images -q ${img_tag} 2> /dev/null)" == "" ]]; then
+  docker build --rm $proxy_args \
+    -t $img_tag .
+fi

docker/ci/test/launch.sh

+docker run --shm-size=8g --privileged --network=host --rm -it -w $PWD -v $PWD:$PWD -v /dataset:/dataset -v /model_zoo:/model_zoo \
+    -v $HOME:$HOME \
+    oneflow-test:0.2 \
+    bash