Smoke test (#346)

* adding smoke test * adding smoke test * fix config.yml * trigger smoke test * test aws erc credential * test aws erc credential * test aws erc credential * test aws erc credential * adding ecr cred * test aws erc credential * test aws erc credential * test aws erc credential - different way * test aws erc credential - different way #2 * test aws erc credential - different way #3 * test aws erc credential - different way #4 * test aws #1 * test aws erc credential, after enabling circleci setting * fixing conda init * add verbose output for smoke test * add docker image build workflow * add docker image build workflow * fix aws ecr login * use images build from circelci * ready for review * updated Readme with instructions on nightly install, also updated docker image which can make nightlies installed via pip pass * nit * reduce size

Smoke test (#346)
* adding smoke test * adding smoke test * fix config.yml * trigger smoke test * test aws erc credential * test aws erc credential * test aws erc credential * test aws erc credential * adding ecr cred * test aws erc credential * test aws erc credential * test aws erc credential - different way * test aws erc credential - different way #2 * test aws erc credential - different way #3 * test aws erc credential - different way #4 * test aws #1 * test aws erc credential, after enabling circleci setting * fixing conda init * add verbose output for smoke test * add docker image build workflow * add docker image build workflow * fix aws ecr login * use images build from circelci * ready for review * updated Readme with instructions on nightly install, also updated docker image which can make nightlies installed via pip pass * nit * reduce size
89a108c7 · Mingbo Wan · GitHub · c90c18d7 · 89a108c7 · 89a108c7
Unverified Commit 89a108c7 authored Nov 25, 2019 by Mingbo Wan Committed by GitHub Nov 25, 2019
Showing with 279 additions and 2 deletions

.circleci/config.yml .circleci/config.yml +154 -1

.circleci/config.yml.in .circleci/config.yml.in +76 -0

.circleci/docker/Dockerfile .circleci/docker/Dockerfile +38 -0

README.md README.md +11 -1

No files found.
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -32,6 +32,14 @@ binary_common: &binary_common
    UNICODE_ABI: << parameters.unicode_abi >>
    CU_VERSION: cpu
+smoke_test_common: &smoke_test_common
+  <<: *binary_common
+  docker:
+    - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
+      aws_auth:
+        aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
+        aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}
 jobs:
  circleci_consistency:
    docker:
@@ -155,6 +163,51 @@ jobs:
              aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
            done
+  smoke_test_linux_conda:
+    <<: *smoke_test_common
+    steps:
+      - run:
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            conda install -y -c pytorch-nightly torchaudio
+            python -c "import torchaudio"
+  smoke_test_linux_pip:
+    <<: *smoke_test_common
+    steps:
+      - run:
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
+            python -c "import torchaudio"
+  docker_image_build:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: large
+    environment:
+      image_name: torchaudio/smoke_test
+    steps:
+      - checkout
+      - run:
+          name: build_docker image
+          no_output_timeout: "1h"
+          command: |
+            cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
+      - run:
+          name: upload docker image
+          no_output_timeout: "1h"
+          command: |
+            set +x
+            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
+            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
+            eval $(aws ecr get-login --region us-east-1 --no-include-email)
+            set -x
+            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
+            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
 workflows:
  build:
    jobs:
@@ -232,6 +285,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_wheel_py2.7
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_wheel_py2.7_smoke_test_pip
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_wheel_py2.7_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_wheel_py2.7_smoke_test_conda
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_wheel_py2.7_upload
          filters:
            branches:
              only: nightly
@@ -247,6 +310,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_wheel_py2.7_unicode
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_wheel_py2.7_unicode_smoke_test_pip
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_wheel_py2.7_unicode_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_wheel_py2.7_unicode_smoke_test_conda
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_wheel_py2.7_unicode_upload
          filters:
            branches:
              only: nightly
@@ -261,6 +334,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_wheel_py3.5
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_wheel_py3.5_smoke_test_pip
+          python_version: "3.5"
+          requires:
+            - nightly_binary_linux_wheel_py3.5_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_wheel_py3.5_smoke_test_conda
+          python_version: "3.5"
+          requires:
+            - nightly_binary_linux_wheel_py3.5_upload
          filters:
            branches:
              only: nightly
@@ -275,6 +358,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_wheel_py3.6
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_wheel_py3.6_smoke_test_pip
+          python_version: "3.6"
+          requires:
+            - nightly_binary_linux_wheel_py3.6_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_wheel_py3.6_smoke_test_conda
+          python_version: "3.6"
+          requires:
+            - nightly_binary_linux_wheel_py3.6_upload
          filters:
            branches:
              only: nightly
@@ -289,6 +382,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_wheel_py3.7
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_wheel_py3.7_smoke_test_pip
+          python_version: "3.7"
+          requires:
+            - nightly_binary_linux_wheel_py3.7_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_wheel_py3.7_smoke_test_conda
+          python_version: "3.7"
+          requires:
+            - nightly_binary_linux_wheel_py3.7_upload
          filters:
            branches:
              only: nightly
@@ -374,6 +477,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_conda_py2.7
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_conda_py2.7_smoke_test_pip
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_conda_py2.7_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_conda_py2.7_smoke_test_conda
+          python_version: "2.7"
+          requires:
+            - nightly_binary_linux_conda_py2.7_upload
          filters:
            branches:
              only: nightly
@@ -388,6 +501,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_conda_py3.5
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_conda_py3.5_smoke_test_pip
+          python_version: "3.5"
+          requires:
+            - nightly_binary_linux_conda_py3.5_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_conda_py3.5_smoke_test_conda
+          python_version: "3.5"
+          requires:
+            - nightly_binary_linux_conda_py3.5_upload
          filters:
            branches:
              only: nightly
@@ -402,6 +525,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_conda_py3.6
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_conda_py3.6_smoke_test_pip
+          python_version: "3.6"
+          requires:
+            - nightly_binary_linux_conda_py3.6_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_conda_py3.6_smoke_test_conda
+          python_version: "3.6"
+          requires:
+            - nightly_binary_linux_conda_py3.6_upload
          filters:
            branches:
              only: nightly
@@ -416,6 +549,16 @@ workflows:
          context: org-member
          requires:
            - nightly_binary_linux_conda_py3.7
+      - smoke_test_linux_pip:
+          name: nightly_binary_linux_conda_py3.7_smoke_test_pip
+          python_version: "3.7"
+          requires:
+            - nightly_binary_linux_conda_py3.7_upload
+      - smoke_test_linux_conda:
+          name: nightly_binary_linux_conda_py3.7_smoke_test_conda
+          python_version: "3.7"
+          requires:
+            - nightly_binary_linux_conda_py3.7_upload
          filters:
            branches:
              only: nightly
@@ -475,3 +618,13 @@ workflows:
          filters:
            branches:
              only: nightly
+  docker_build:
+    triggers:
+      - schedule:
+          cron: "0 10 * * 0"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - docker_image_build
\ No newline at end of file
--- a/.circleci/config.yml.in
+++ b/.circleci/config.yml.in
@@ -32,6 +32,14 @@ binary_common: &binary_common
    UNICODE_ABI: << parameters.unicode_abi >>
    CU_VERSION: cpu
+smoke_test_common: &smoke_test_common
+  <<: *binary_common
+  docker:
+    - image: 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:d26af7d0-8458-face-book-84239b5c75c0
+      aws_auth:
+        aws_access_key_id: ${ECR_AWS_ACCESS_KEY}
+        aws_secret_access_key: ${ECR_AWS_SECRET_ACCESS_KEY}
 jobs:
  circleci_consistency:
    docker:
@@ -155,6 +163,50 @@ jobs:
              aws s3 cp "$pkg" "s3://pytorch/whl/nightly/" --acl public-read
            done
+  smoke_test_linux_conda:
+    <<: *smoke_test_common
+    steps:
+      - run:
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            conda install -y -c pytorch-nightly torchaudio
+            python -c "import torchaudio"
+  smoke_test_linux_pip:
+    <<: *smoke_test_common
+    steps:
+      - run:
+          command: |
+            set -x
+            source /usr/local/etc/profile.d/conda.sh && conda activate python${PYTHON_VERSION}
+            pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
+            python -c "import torchaudio"
+  docker_image_build:
+    machine:
+      image: ubuntu-1604:201903-01
+    resource_class: large
+    environment:
+      image_name: torchaudio/smoke_test
+    steps:
+      - checkout
+      - run:
+          name: build_docker image
+          no_output_timeout: "1h"
+          command: |
+            cd .circleci/docker && docker build . -t ${image_name}:${CIRCLE_WORKFLOW_ID}
+      - run:
+          name: upload docker image
+          no_output_timeout: "1h"
+          command: |
+            set +x
+            export AWS_ACCESS_KEY_ID=${ECR_AWS_ACCESS_KEY}
+            export AWS_SECRET_ACCESS_KEY=${ECR_AWS_SECRET_ACCESS_KEY}
+            eval $(aws ecr get-login --region us-east-1 --no-include-email)
+            set -x
+            docker tag ${image_name}:${CIRCLE_WORKFLOW_ID} 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
+            docker push 308535385114.dkr.ecr.us-east-1.amazonaws.com/${image_name}:${CIRCLE_WORKFLOW_ID}
 {%- macro workflow(btype, os, python_version, unicode, prefix='', filter_branch=None, upload=False) %}
      - binary_{{os}}_{{btype}}:
@@ -175,6 +227,20 @@ jobs:
          context: org-member
          requires:
            - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}
+{%-       if filter_branch == 'nightly' %}
+{%-         if os == 'linux' %}
+      - smoke_test_linux_pip:
+          name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_smoke_test_pip
+          python_version: "{{python_version}}"
+          requires:
+            - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload
+      - smoke_test_linux_conda:
+          name: {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_smoke_test_conda
+          python_version: "{{python_version}}"
+          requires:
+            - {{prefix}}binary_{{os}}_{{btype}}_py{{python_version}}{{ "_unicode" if unicode }}_upload
+{%-         endif %}
+{%-       endif %}
 {%-   if filter_branch %}
          filters:
            branches:
@@ -207,3 +273,13 @@ workflows:
    jobs:
      - circleci_consistency
      {{ workflows(prefix="nightly_", filter_branch="nightly", upload=True) }}
+  docker_build:
+    triggers:
+      - schedule:
+          cron: "0 10 * * 0"
+          filters:
+            branches:
+              only:
+                - master
+    jobs:
+      - docker_image_build
--- a/.circleci/docker/Dockerfile
+++ b/.circleci/docker/Dockerfile
+# this Dockerfile is for torchaudio smoke test, it will be created periodically via CI system
+# if you need to do it locally, follow below steps once you have Docker installed
+# assuming you're within the directory where this Dockerfile located
+#  $ docker build . -t torchaudio/smoketest
+# if you want to push to aws ecr, make sure you have the rights to write to ECR, then run
+# $ eval $(aws ecr get-login --region us-east-1 --no-include-email)
+# $ export MYTAG=localbuild  ## you can choose whatever tag you like
+# $ docker tag torchaudio/smoketest 308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
+# $ docker push  308535385114.dkr.ecr.us-east-1.amazonaws.com/torchaudio/smoke_test:${MYTAG}
+FROM ubuntu:latest
+RUN apt-get -qq update && apt-get -qq -y install curl bzip2 sox libsox-dev libsox-fmt-all \
+    && curl -sSL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh \
+    && bash /tmp/miniconda.sh -bfp /usr/local \
+    && rm -rf /tmp/miniconda.sh \
+    && conda install -y python=3 \
+    && conda update conda \
+    && apt-get -qq -y remove curl bzip2 \
+    && apt-get -qq -y autoremove \
+    && apt-get autoclean \
+    && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log \
+    && conda clean --all --yes
+ENV PATH /opt/conda/bin:$PATH
+RUN conda create -y --name python2.7 python=2.7
+RUN conda create -y --name python3.5 python=3.5
+RUN conda create -y --name python3.6 python=3.6
+RUN conda create -y --name python3.7 python=3.7
+SHELL [ "/bin/bash", "-c" ]
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python2.7 && conda install -y -c conda-forge sox && conda install -y numpy future
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.5 && conda install -y -c conda-forge sox && conda install -y numpy six
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.6 && conda install -y -c conda-forge sox && conda install -y numpy
+RUN source /usr/local/etc/profile.d/conda.sh && conda activate python3.7 && conda install -y -c conda-forge sox && conda install -y numpy
+RUN echo "source /usr/local/etc/profile.d/conda.sh" >> ~/.bashrc
+CMD [ "/bin/bash"]
--- a/README.md
+++ b/README.md
@@ -66,12 +66,22 @@ before running this command.)
 At the moment, there is no automated nightly build process, but we occasionally
 build nightlies based on PyTorch nightlies by hand following the instructions in
-[packaging](packaging).  To install the latest nightly, run:
+[packaging](packaging).  To install the latest nightly via pip, run:
 ```
+pip install numpy
+pip install future # only on python 2.7
+pip install six    # only on python 3.5
 pip install torchaudio_nightly -f https://download.pytorch.org/whl/nightly/torch_nightly.html
 ```
+To install the latest nightly via conda, run:
+```
+conda install -y -c pytorch-nightly torchaudio
+```
 ### From Source
 If your system configuration is not among the supported configurations