Setup: Add Test - Add Codecov (#9)

Add code coverage configuration.

.editorconfig

@@ -11,7 +11,7 @@ insert_final_newline = true
 [*.py]
 max_line_length = 120
 
-[*.yaml]
+[*.{yml,yaml}]
 indent_size = 2
 
 [*.json]

README.md

@@ -2,6 +2,7 @@
 
 [![Build Status](https://dev.azure.com/msrasrg/SuperBenchmark/_apis/build/status/microsoft.superbenchmark?branchName=dev)](https://dev.azure.com/msrasrg/SuperBenchmark/_build?definitionId=77)
 [![Lint](https://github.com/microsoft/superbenchmark/workflows/Lint/badge.svg)](https://github.com/microsoft/superbenchmark/actions?query=workflow%3ALint)
+[![Codecov](https://codecov.io/gh/microsoft/superbenchmark/branch/dev/graph/badge.svg?token=DDiDLW7pSd)](https://codecov.io/gh/microsoft/superbenchmark/branch/dev)
 
 
 SuperBench is a benchmarking and diagnosis tool for AI infrastructure,
@@ -144,7 +145,7 @@ SuperBenchmark is an open-source project. Your participation and contribution ar
 
 1. Bug fixes for existing features.
 2. New features for benchmark module (micro-benchmark, model-benchmark, etc.)
-   
+
    If you would like to contribute a new feature on SuperBenchmark, please submit your proposal first. In [GitHub Issues](https://github.com/microsoft/superbenchmark/issues) module, choose `Enhancement Request` to finish the submission. If the proposal is accepted, you can submit pull request to origin dev branch.
 
 #### Contribution steps

azure-pipelines.yml

@@ -7,13 +7,14 @@
 # https://aka.ms/yaml
 
 trigger:
-- master
-- dev
+  - master
+  - dev
 
 resources:
-- repo: self
+  - repo: self
 
 variables:
+  imageName: test
   tag: '$(Build.BuildId)'
 
 steps:
@@ -22,5 +23,12 @@ steps:
     inputs:
       command: build
       dockerfile: '$(Build.SourcesDirectory)/test.dockerfile'
+      repository: $(imageName)
       tags: |
         $(tag)
+  - script: |
+      ci_env=`bash <(curl -s https://codecov.io/env)`
+      docker run $ci_env --entrypoint /bin/bash $(imageName):$(tag) -c 'bash <(curl -s https://codecov.io/bash)'
+    displayName: Report coverage results
+    env:
+      CODECOV_TOKEN: $(codecovToken)

codecov.yml

+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        target: 80%
+        threshold: 1%
+        branches:
+          - main
+          - dev

setup.py

@@ -99,7 +99,7 @@ def finalize_options(self):
 
     def run(self):
         """Run pytest."""
-        errno = os.system('python3 -m pytest -v')
+        errno = os.system('python3 -m pytest -v --cov=superbench --cov-report=xml --cov-report=term-missing tests/')
         sys.exit(0 if errno == 0 else 1)
 
 
@@ -143,6 +143,7 @@ def run(self):
             'flake8-docstrings>=1.5.0',
             'pydocstyle>=5.1.1',
             'pytest>=6.2.2',
+            'pytest-cov>=2.11.1',
         ],
     },
     package_data={},

test.dockerfile

@@ -5,6 +5,8 @@ FROM ubuntu:18.04
 
 # Install the python3.7 and pip
 RUN apt-get update && apt-get install -y \
+    git \
+    curl \
     python3.7-dev \
     python3-pip
 

tests/test_example.py

@@ -6,6 +6,8 @@
 Get it from https://docs.pytest.org/en/stable/.
 """
 
+import superbench
+
 
 def inc(x):
     """Increase an integer.
@@ -22,3 +24,9 @@ def inc(x):
 def test_answer():
     """Test inc function."""
     assert inc(3) == 4
+
+
+def test_superbench():
+    """Test SuperBench."""
+    assert (superbench.__version__ == '0.0.0')
+    assert (superbench.__author__ == 'Microsoft')