build and upload docs from circleci (#3259)

Co-authored-by: Francisco Massa <fvsmassa@gmail.com>

.circleci/build_docs/commit_docs.sh

+#!/usr/bin/env bash
+
+set -ex
+
+
+if [ "$2" == "" ]; then
+    echo call as "$0" "<src>" "<target branch>"
+    echo where src is the root of the built documentation git checkout and
+    echo branch should be "master" or "1.7" or so
+    exit 1
+fi
+
+src=$1
+target=$2
+
+echo "committing docs from ${src} to ${target}"
+
+pushd "${src}"
+git checkout gh-pages
+mkdir -p ./"${target}"
+rm -rf ./"${target}"/*
+cp -r "${src}/docs/build/html/"* ./"$target"
+if [ "${target}" == "master" ]; then
+    mkdir -p ./_static
+    rm -rf ./_static/*
+    cp -r "${src}/docs/build/html/_static/"* ./_static
+    git add --all ./_static || true
+fi
+git add --all ./"${target}" || true
+git config user.email "soumith+bot@pytorch.org"
+git config user.name "pytorchbot"
+# If there aren't changes, don't make a commit; push is no-op
+git commit -m "auto-generating sphinx docs" || true
+git remote add https https://github.com/pytorch/vision.git
+git push -u https gh-pages

.circleci/config.yml

@@ -48,7 +48,7 @@ commands:
 
 binary_common: &binary_common
   parameters:
-    # Edit these defaults to do a release`
+    # Edit these defaults to do a release
     build_version:
       description: "version number of release binary; by default, build a nightly"
       type: string
@@ -705,6 +705,71 @@ jobs:
             packaging/windows/internal/cuda_install.bat
             packaging/build_cmake.sh
 
+  build_docs:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - checkout
+      - run:
+          name: Setup
+          command: .circleci/unittest/linux/scripts/setup_env.sh
+      - designate_upload_channel
+      - run:
+          name: Install torchvision
+          command: .circleci/unittest/linux/scripts/install.sh
+      - run:
+          name: Build docs
+          command: |
+            set -ex
+            tag=${CIRCLE_TAG:1:5}
+            VERSION=${tag:-master}
+            eval "$(./conda/bin/conda shell.bash hook)"
+            conda activate ./env
+            pushd docs
+            pip install -r requirements.txt
+            make html
+            popd
+      - persist_to_workspace:
+          root: ./
+          paths:
+            - "*"
+      - store_artifacts:
+          path: ./docs/build/html
+          destination: docs
+
+  upload_docs:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run:
+          name: Generate netrc
+          command: |
+            # set credentials for https pushing
+            # requires the org-member context
+            cat > ~/.netrc \<<DONE
+              machine github.com
+              login pytorchbot
+              password ${GITHUB_PYTORCHBOT_TOKEN}
+            DONE
+      - run:
+          name: Upload docs
+          command: |
+            # Don't use "checkout" step since it uses ssh, which cannot git push
+            # https://circleci.com/docs/2.0/configuration-reference/#checkout
+            set -ex
+            tag=${CIRCLE_TAG:1:5}
+            target=${tag:-master}
+            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
+
+
 workflows:
   build:
     jobs:
@@ -1085,6 +1150,21 @@ workflows:
           cu_version: cu110
           name: binary_win_conda_py3.8_cu110
           python_version: '3.8'
+      - build_docs:
+          name: build_docs
+          python_version: '3.7'
+          requires:
+          - binary_linux_wheel_py3.7_cpu
+      - upload_docs:
+          context: org-member
+          filters:
+            branches:
+              only:
+              - nightly
+          name: upload_docs
+          python_version: '3.7'
+          requires:
+          - build_docs
       - python_lint
       - python_type_check
       - clang_format

.circleci/config.yml.in

@@ -48,7 +48,7 @@ commands:
 
 binary_common: &binary_common
   parameters:
-    # Edit these defaults to do a release`
+    # Edit these defaults to do a release
     build_version:
       description: "version number of release binary; by default, build a nightly"
       type: string
@@ -705,6 +705,71 @@ jobs:
             packaging/windows/internal/cuda_install.bat
             packaging/build_cmake.sh
 
+  build_docs:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - checkout
+      - run:
+          name: Setup
+          command: .circleci/unittest/linux/scripts/setup_env.sh
+      - designate_upload_channel
+      - run:
+          name: Install torchvision
+          command: .circleci/unittest/linux/scripts/install.sh
+      - run:
+          name: Build docs
+          command: |
+            set -ex
+            tag=${CIRCLE_TAG:1:5}
+            VERSION=${tag:-master}
+            eval "$(./conda/bin/conda shell.bash hook)"
+            conda activate ./env
+            pushd docs
+            pip install -r requirements.txt
+            make html
+            popd
+      - persist_to_workspace:
+          root: ./
+          paths:
+            - "*"
+      - store_artifacts:
+          path: ./docs/build/html
+          destination: docs
+
+  upload_docs:
+    <<: *binary_common
+    docker:
+      - image: "pytorch/manylinux-cuda100"
+    resource_class: 2xlarge+
+    steps:
+      - attach_workspace:
+          at: ~/workspace
+      - run:
+          name: Generate netrc
+          command: |
+            # set credentials for https pushing
+            # requires the org-member context
+            cat > ~/.netrc \<<DONE
+              machine github.com
+              login pytorchbot
+              password ${GITHUB_PYTORCHBOT_TOKEN}
+            DONE
+      - run:
+          name: Upload docs
+          command: |
+            # Don't use "checkout" step since it uses ssh, which cannot git push
+            # https://circleci.com/docs/2.0/configuration-reference/#checkout
+            set -ex
+            tag=${CIRCLE_TAG:1:5}
+            target=${tag:-master}
+            ~/workspace/.circleci/build_docs/commit_docs.sh ~/workspace $target
+
+
 workflows:
   build:
 {%- if True %}

.circleci/regenerate.py

@@ -43,6 +43,10 @@ def build_workflows(prefix='', filter_branch=None, upload=False, indentation=6,
                             btype, os_type, python_version, cu_version,
                             unicode, prefix, upload, filter_branch=fb)
 
+    if not filter_branch:
+        # Build on every pull request, but upload only on nightly and tags
+        w += build_doc_job(None)
+        w += upload_doc_job('nightly')
     return indent(indentation, w)
 
 
@@ -65,6 +69,31 @@ def workflow_pair(btype, os_type, python_version, cu_version, unicode, prefix=''
     return w
 
 
+def build_doc_job(filter_branch):
+    job = {
+        "name": "build_docs",
+        "python_version": "3.7",
+        "requires": ["binary_linux_wheel_py3.7_cpu", ],
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    return [{"build_docs": job}]
+
+
+def upload_doc_job(filter_branch):
+    job = {
+        "name": "upload_docs",
+        "context": "org-member",
+        "python_version": "3.7",
+        "requires": ["build_docs", ],
+    }
+
+    if filter_branch:
+        job["filters"] = gen_filter_branch_tree(filter_branch)
+    return [{"upload_docs": job}]
+
+
 manylinux_images = {
     "cu92": "pytorch/manylinux-cuda92",
     "cu101": "pytorch/manylinux-cuda101",