Skip to content

GitLab

Unverified Commit 89a108c7 authored by Mingbo Wan's avatar Mingbo Wan Committed by GitHub
Browse files

Smoke test (#346) 

* adding smoke test

* adding smoke test

* fix config.yml

* trigger smoke test

* test aws erc credential

* test aws erc credential

* test aws erc credential

* test aws erc credential

* adding ecr cred

* test aws erc credential

* test aws erc credential

* test aws erc credential - different way

* test aws erc credential - different way #2

* test aws erc credential - different way #3

* test aws erc credential - different way #4

* test aws #1

* test aws erc credential, after enabling circleci setting

* fixing conda init

* add verbose output for smoke test

* add docker image build workflow

* add docker image build workflow

* fix aws ecr login

* use images build from circelci

* ready for review

* updated Readme with instructions on nightly install, also updated docker image which can make nightlies installed via pip pass

* nit

* reduce size
parent c90c18d7
Hide whitespace changes
Inline Side-by-side
Showing with 279 additions and 2 deletions
.circleci/config.yml
View file @ 89a108c7
......@@ -32,6 +32,14 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: cpu
smoke_test_common: &smoke_test_common
<<: *binary_common
docker:
- image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
aws_auth:
aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}
jobs:
circleci_consistency:
docker:
......@@ -155,6 +163,51 @@ jobs:
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
done
smoke_test_linux_conda:
<<: *smoke_test_common
steps:
- run:
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -y -c pytorch-nightly torchaudio
python -c "import torchaudio"
smoke_test_linux_pip:
<<: *smoke_test_common
steps:
- run:
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
python -c "import torchaudio"
docker_image_build:
machine:
image: ubuntu-1604:201903-01
resource_class: large
environment:
image_name: torchaudio/smoke_test
steps:
- checkout
- run:
name: build_docker image
no_output_timeout: "1h"
command: |
cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
- run:
name: upload docker image
no_output_timeout: "1h"
command: |
set +x
export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
eval $(aws ecr get-login --region us-east-1 --no-include-email)
set -x
docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
workflows:
build:
jobs:
......@@ -232,6 +285,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7
- smoke_test_linux_pip:
name: nightly_binary_linux_wheel_py2.7_smoke_test_pip
python_version: "2.7"
requires:
- nightly_binary_linux_wheel_py2.7_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_wheel_py2.7_smoke_test_conda
python_version: "2.7"
requires:
- nightly_binary_linux_wheel_py2.7_upload
filters:
branches:
only: nightly
......@@ -247,6 +310,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_wheel_py2.7_unicode
- smoke_test_linux_pip:
name: nightly_binary_linux_wheel_py2.7_unicode_smoke_test_pip
python_version: "2.7"
requires:
- nightly_binary_linux_wheel_py2.7_unicode_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_wheel_py2.7_unicode_smoke_test_conda
python_version: "2.7"
requires:
- nightly_binary_linux_wheel_py2.7_unicode_upload
filters:
branches:
only: nightly
......@@ -261,6 +334,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_wheel_py3.5
- smoke_test_linux_pip:
name: nightly_binary_linux_wheel_py3.5_smoke_test_pip
python_version: "3.5"
requires:
- nightly_binary_linux_wheel_py3.5_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_wheel_py3.5_smoke_test_conda
python_version: "3.5"
requires:
- nightly_binary_linux_wheel_py3.5_upload
filters:
branches:
only: nightly
......@@ -275,6 +358,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_wheel_py3.6
- smoke_test_linux_pip:
name: nightly_binary_linux_wheel_py3.6_smoke_test_pip
python_version: "3.6"
requires:
- nightly_binary_linux_wheel_py3.6_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_wheel_py3.6_smoke_test_conda
python_version: "3.6"
requires:
- nightly_binary_linux_wheel_py3.6_upload
filters:
branches:
only: nightly
......@@ -289,6 +382,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_wheel_py3.7
- smoke_test_linux_pip:
name: nightly_binary_linux_wheel_py3.7_smoke_test_pip
python_version: "3.7"
requires:
- nightly_binary_linux_wheel_py3.7_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_wheel_py3.7_smoke_test_conda
python_version: "3.7"
requires:
- nightly_binary_linux_wheel_py3.7_upload
filters:
branches:
only: nightly
......@@ -374,6 +477,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_conda_py2.7
- smoke_test_linux_pip:
name: nightly_binary_linux_conda_py2.7_smoke_test_pip
python_version: "2.7"
requires:
- nightly_binary_linux_conda_py2.7_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_conda_py2.7_smoke_test_conda
python_version: "2.7"
requires:
- nightly_binary_linux_conda_py2.7_upload
filters:
branches:
only: nightly
......@@ -388,6 +501,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_conda_py3.5
- smoke_test_linux_pip:
name: nightly_binary_linux_conda_py3.5_smoke_test_pip
python_version: "3.5"
requires:
- nightly_binary_linux_conda_py3.5_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_conda_py3.5_smoke_test_conda
python_version: "3.5"
requires:
- nightly_binary_linux_conda_py3.5_upload
filters:
branches:
only: nightly
......@@ -402,6 +525,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_conda_py3.6
- smoke_test_linux_pip:
name: nightly_binary_linux_conda_py3.6_smoke_test_pip
python_version: "3.6"
requires:
- nightly_binary_linux_conda_py3.6_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_conda_py3.6_smoke_test_conda
python_version: "3.6"
requires:
- nightly_binary_linux_conda_py3.6_upload
filters:
branches:
only: nightly
......@@ -416,6 +549,16 @@ workflows:
context: org-member
requires:
- nightly_binary_linux_conda_py3.7
- smoke_test_linux_pip:
name: nightly_binary_linux_conda_py3.7_smoke_test_pip
python_version: "3.7"
requires:
- nightly_binary_linux_conda_py3.7_upload
- smoke_test_linux_conda:
name: nightly_binary_linux_conda_py3.7_smoke_test_conda
python_version: "3.7"
requires:
- nightly_binary_linux_conda_py3.7_upload
filters:
branches:
only: nightly
......@@ -474,4 +617,14 @@ workflows:
- nightly_binary_macos_conda_py3.7
filters:
branches:
only: nightly
\ No newline at end of file
only: nightly
docker_build:
triggers:
- schedule:
cron: "0 10 * * 0"
filters:
branches:
only:
- master
jobs:
- docker_image_build
\ No newline at end of file
.circleci/config.yml.in
View file @ 89a108c7
......@@ -32,6 +32,14 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: cpu
smoke_test_common: &smoke_test_common
<<: *binary_common
docker:
- image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
aws_auth:
aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}
jobs:
circleci_consistency:
docker:
......@@ -155,6 +163,50 @@ jobs:
aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
done
smoke_test_linux_conda:
<<: *smoke_test_common
steps:
- run:
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
conda install -y -c pytorch-nightly torchaudio
python -c "import torchaudio"
smoke_test_linux_pip:
<<: *smoke_test_common
steps:
- run:
command: |
set -x
source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
python -c "import torchaudio"
docker_image_build:
machine:
image: ubuntu-1604:201903-01
resource_class: large
environment:
image_name: torchaudio/smoke_test
steps:
- checkout
- run:
name: build_docker image
no_output_timeout: "1h"
command: |
cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
- run:
name: upload docker image
no_output_timeout: "1h"
command: |
set +x
export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
eval $(aws ecr get-login --region us-east-1 --no-include-email)
set -x
docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
{%- macro workflow(btype, os, python_version, unicode, prefix='', filter_branch=None, upload=False) %}
- binary_{{os}}_{{btype}}:
......@@ -175,6 +227,20 @@ jobs:
context: org-member
requires:
- {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}
{%- if filter_branch == 'nightly' %}
{%- if os == 'linux' %}
- smoke_test_linux_pip:
name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_smoke_test_pip
python_version: "{{python_version}}"
requires:
- {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload
- smoke_test_linux_conda:
name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_smoke_test_conda
python_version: "{{python_version}}"
requires:
- {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload
{%- endif %}
{%- endif %}
{%- if filter_branch %}
filters:
branches:
......@@ -207,3 +273,13 @@ workflows:
jobs:
- circleci_consistency
{{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
docker_build:
triggers:
- schedule:
cron: "0 10 * * 0"
filters:
branches:
only:
- master
jobs:
- docker_image_build
.circleci/docker/Dockerfile 0 → 100644
View file @ 89a108c7
# this Dockerfile is for torchaudio smoke test, it will be created periodically via CI system
# if you need to do it locally, follow below steps once you have Docker installed
# assuming you're within the directory where this Dockerfile located
# $ docker build . -t torchaudio/smoketest
# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run
# $ eval $(aws ecr get-login --region us-east-1 --no-include-email)
# $ export MYTAG=localbuild ## you can choose whatever tag you like
# $ docker tag torchaudio/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
# $ docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
FROM ubuntu:latest
RUN apt-get -qq update && apt-get -qq -y install curl bzip2 sox libsox-dev libsox-fmt-all \
&& curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
&& bash /tmp/miniconda.sh -bfp /usr/local \
&& rm -rf /tmp/miniconda.sh \
&& conda install -y python=3 \
&& conda update conda \
&& apt-get -qq -y remove curl bzip2 \
&& apt-get -qq -y autoremove \
&& apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
&& conda clean --all --yes
ENV PATH /opt/conda/bin:$PATH
RUN conda create -y --name python2.7 python=2.7
RUN conda create -y --name python3.5 python=3.5
RUN conda create -y --name python3.6 python=3.6
RUN conda create -y --name python3.7 python=3.7
SHELL [ "/bin/bash", "-c" ]
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python2.7 && conda install -y -c conda-forge sox && conda install -y numpy future
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.5 && conda install -y -c conda-forge sox && conda install -y numpy six
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.6 && conda install -y -c conda-forge sox && conda install -y numpy
RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.7 && conda install -y -c conda-forge sox && conda install -y numpy
RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc
CMD [ "/bin/bash"]
README.md
View file @ 89a108c7
......@@ -66,12 +66,22 @@ before running this command.)
At the moment, there is no automated nightly build process, but we occasionally
build nightlies based on PyTorch nightlies by hand following the instructions in
[packaging](packaging). To install the latest nightly, run:
[packaging](packaging). To install the latest nightly via pip, run:
```
pip install numpy
pip install future # only on python 2.7
pip install six # only on python 3.5
pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
```
To install the latest nightly via conda, run:
```
conda install -y -c pytorch-nightly torchaudio
```
### From Source
If your system configuration is not among the supported configurations
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment