config.yml.in 7.65 KB
Newer Older
1
2
3
4
version: 2.1

# How to test the Linux jobs:
#   - Install CircleCI local CLI: https://circleci.com/docs/2.0/local-cli/
Edward Z. Yang's avatar
Edward Z. Yang committed
5
6
7
#   - circleci config process .circleci/config.yml > gen.yml && circleci local execute -c gen.yml --job binary_linux_wheel_py3.7
#     - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
#       Job names are 'name:' key.
8
9
10

binary_common: &binary_common
  parameters:
Edward Z. Yang's avatar
Edward Z. Yang committed
11
    # Edit these defaults to do a release
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
    build_version:
      description: "version number of release binary; by default, build a nightly"
      type: string
      default: ""
    pytorch_version:
      description: "PyTorch version to build against; by default, use a nightly"
      type: string
      default: ""
    # Don't edit these
    python_version:
      description: "Python version to build against (e.g., 3.7)"
      type: string
    unicode_abi:
      description: "Python 2.7 wheel only: whether or not we are cp27mu (default: no)"
      type: string
      default: ""
  environment:
    PYTHON_VERSION: << parameters.python_version >>
    BUILD_VERSION: << parameters.build_version >>
    PYTORCH_VERSION: << parameters.pytorch_version >>
    UNICODE_ABI: << parameters.unicode_abi >>
Edward Z. Yang's avatar
Edward Z. Yang committed
33
    CU_VERSION: cpu
34

Mingbo Wan's avatar
Mingbo Wan committed
35
36
37
38
39
40
41
42
smoke_test_common: &smoke_test_common
  <<: *binary_common
  docker:
    - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
      aws_auth:
        aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
        aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}

43
44
45
46
47
48
49
50
jobs:
  circleci_consistency:
    docker:
      - image: circleci/python:3.7
    steps:
      - checkout
      - run:
          command: |
51
            pip install --user --progress-bar off jinja2 pyyaml
52
53
54
55
56
57
            python .circleci/regenerate.py
            git diff --exit-code || (echo ".circleci/config.yml not in sync with config.yml.in! Run .circleci/regenerate.py to update config"; exit 1)

  binary_linux_wheel:
    <<: *binary_common
    docker:
58
      - image: "pytorch/manylinux-cuda100"
59
60
61
62
63
64
    resource_class: 2xlarge+
    steps:
      - checkout
      - run: packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
65
66
67
68
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
69
70
71
72

  binary_linux_conda:
    <<: *binary_common
    docker:
73
      - image: "pytorch/conda-cuda"
74
75
76
77
78
79
    resource_class: 2xlarge+
    steps:
      - checkout
      - run: packaging/build_conda.sh
      - store_artifacts:
          path: /opt/conda/conda-bld/linux-64
Edward Z. Yang's avatar
Edward Z. Yang committed
80
81
82
83
      - persist_to_workspace:
          root: /opt/conda/conda-bld/linux-64
          paths:
            - "*"
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

  binary_macos_wheel:
    <<: *binary_common
    macos:
      xcode: "9.0"
    steps:
      - checkout
      - run:
          # Cannot easily deduplicate this as source'ing activate
          # will set environment variables which we need to propagate
          # to build_wheel.sh
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            packaging/build_wheel.sh
      - store_artifacts:
          path: dist
Edward Z. Yang's avatar
Edward Z. Yang committed
102
103
104
105
      - persist_to_workspace:
          root: dist
          paths:
            - "*"
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

  binary_macos_conda:
    <<: *binary_common
    macos:
      xcode: "9.0"
    steps:
      - checkout
      - run:
          command: |
            curl -o conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh
            sh conda.sh -b
            source $HOME/miniconda3/bin/activate
            conda install -yq conda-build
            packaging/build_conda.sh
      - store_artifacts:
          path: /Users/distiller/miniconda3/conda-bld/osx-64
Edward Z. Yang's avatar
Edward Z. Yang committed
122
123
124
125
      - persist_to_workspace:
          root: /Users/distiller/miniconda3/conda-bld/osx-64
          paths:
            - "*"
126

Edward Z. Yang's avatar
Edward Z. Yang committed
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
  # Requires org-member context
  binary_conda_upload:
    docker:
      - image: continuumio/miniconda
    steps:
      - attach_workspace:
          at: ~/workspace
      - run:
          command: |
            # Prevent credential from leaking
            conda install -yq anaconda-client
            set +x
            anaconda login \
                --username "$PYTORCH_BINARY_PJH5_CONDA_USERNAME" \
                --password "$PYTORCH_BINARY_PJH5_CONDA_PASSWORD"
            set -x
            anaconda upload ~/workspace/*.tar.bz2 -u pytorch-nightly --label main --no-progress --force

  # Requires org-member context
  binary_wheel_upload:
    docker:
      - image: circleci/python:3.7
    steps:
      - attach_workspace:
          at: ~/workspace
      - checkout
      - run:
          command: |
            pip install --user awscli
            export PATH="$HOME/.local/bin:$PATH"
            # Prevent credential from leaking
            set +x
            export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}"
            set -x
            for pkg in ~/workspace/*.whl; do
              aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
            done

Mingbo Wan's avatar
Mingbo Wan committed
166
167
168
  smoke_test_linux_conda:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
169
170
      - attach_workspace:
          at: ~/workspace
Mingbo Wan's avatar
Mingbo Wan committed
171
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
172
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
173
174
175
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
176
177
178
179
180
181
            conda install -v -y -c pytorch-nightly pytorch
            conda install -v -y $(ls ~/workspace/torchaudio*.tar.bz2)
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
182
183
184
185
186
            python -c "import torchaudio"

  smoke_test_linux_pip:
    <<: *smoke_test_common
    steps:
Mingbo Wan's avatar
Mingbo Wan committed
187
188
      - attach_workspace:
          at: ~/workspace
Mingbo Wan's avatar
Mingbo Wan committed
189
      - run:
Mingbo Wan's avatar
Mingbo Wan committed
190
          name: install binaries
Mingbo Wan's avatar
Mingbo Wan committed
191
192
193
          command: |
            set -x
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
194
195
196
197
198
            pip install $(ls ~/workspace/torchaudio*.whl) -f https://download.pytorch.org/whl/nightly/torch_nightly.html
      - run:
          name: smoke test
          command: |
            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
Mingbo Wan's avatar
Mingbo Wan committed
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
            python -c "import torchaudio"

  docker_image_build:
    machine:
      image: ubuntu-1604:201903-01
    resource_class: large
    environment:
      image_name: torchaudio/smoke_test
    steps:
      - checkout
      - run:
          name: build_docker image
          no_output_timeout: "1h"
          command: |
            cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
      - run:
          name: upload docker image
          no_output_timeout: "1h"
          command: |
            set +x
            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
            eval $(aws ecr get-login --region us-east-1 --no-include-email)
            set -x
            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
Edward Z. Yang's avatar
Edward Z. Yang committed
225
226
227
228
229
230
231
232
233
234
235
236

workflows:
  build:
{%- if True %}
    jobs:
      - circleci_consistency
      {{ workflows() }}

  nightly:
{%- endif %}
    jobs:
      - circleci_consistency
Edward Z. Yang's avatar
Edward Z. Yang committed
237
      {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
Mingbo Wan's avatar
Mingbo Wan committed
238
239
240
241
242
243
244
245
246
247
  docker_build:
    triggers:
      - schedule:
          cron: "0 10 * * 0"
          filters:
            branches:
              only:
                - master
    jobs:
      - docker_image_build