Dockerfile download onnx models to enable real model unit tests (#628)

* turn on the alexnet unit tests through downloading the real model in dockerfile

* clang format

* refine home directory

* enable all real model unit tests

* clang format

* fix review comments and disable incorrect test cases

* clang format

* increase timeout value to avoid time out

* turn on one more test

* disable one more real model unit test

* fix review comments

* remove unnecessary comment lines

* clang format

* fix review comments

* print out program info when there is an error

* clang format

* redirect c++ stdout to sys.stdout of python for python api

* clang format

* two minor issues

* fix a cppcheck error

* fix review comments

* clang format

* remove unnecessary changes

* refine script

Dockerfile

@@ -92,6 +92,11 @@ RUN cget -p $PREFIX install -f /dev-requirements.txt -DMIOPEN_CACHE_DIR=""
 
 RUN pip3 install onnx==1.7.0 numpy==1.18.5 typing==3.7.4 pytest==6.0.1
 
+# Download real models to run onnx unit tests
+ENV ONNX_HOME=$HOME
+COPY ./tools/download_models.sh /
+RUN chmod +x /download_models.sh && /download_models.sh && rm /download_models.sh
+
 # Install newer cmake for onnx runtime
 RUN cget -p /opt/cmake install kitware/cmake@v3.13.0
 

src/py/backend/backend.py

@@ -23,6 +23,7 @@ def get_device():
 class MIGraphXBackend(Backend):
     _device = "GPU"
     _input_names = []
+    _prog_string = ""
 
     @classmethod
     def set_device(cls, device):
@@ -38,6 +39,10 @@ class MIGraphXBackend(Backend):
     Note: This is not the official Python API.
     """  # noqa: E501
 
+    @classmethod
+    def get_program(cls):
+        return cls._prog_string
+
     @classmethod
     def is_compatible(cls, model, device=None, **kwargs):
         """
@@ -85,9 +90,13 @@ class MIGraphXBackend(Backend):
                     "Incompatible device expected '{0}', got '{1}'".format(
                         device, get_device()))
             inf = migraphx.parse_onnx_buffer(model)
+            cls._prog_string = str("\nProgram =\n{}".format(inf))
             device = cls._device
             cls._input_names = inf.get_parameter_names()
             inf.compile(migraphx.get_target(device.lower()))
+            cls._prog_string = cls._prog_string + str(
+                "\nCompiled program =\n{}".format(inf))
+
             return cls.prepare(inf, device, **kwargs)
         else:
             # type: ModelProto

test/py/onnx_backend_test.py

@@ -20,16 +20,24 @@ class MIGraphXBackendTest(onnx.backend.test.BackendTest):
 
     @classmethod
     def assert_similar_outputs(cls, ref_outputs, outputs, rtol, atol):
-        np.testing.assert_equal(len(ref_outputs), len(outputs))
+        prog_string = c2.get_program()
+        np.testing.assert_equal(len(ref_outputs),
+                                len(outputs),
+                                err_msg=prog_string)
         for i in range(len(outputs)):
-            np.testing.assert_equal(ref_outputs[i].dtype, outputs[i].dtype)
+            np.testing.assert_equal(ref_outputs[i].dtype,
+                                    outputs[i].dtype,
+                                    err_msg=prog_string)
             if ref_outputs[i].dtype == np.object:
-                np.testing.assert_array_equal(ref_outputs[i], outputs[i])
+                np.testing.assert_array_equal(ref_outputs[i],
+                                              outputs[i],
+                                              err_msg=prog_string)
             else:
                 np.testing.assert_allclose(ref_outputs[i],
                                            outputs[i],
                                            rtol=1e-3,
-                                           atol=1e-5)
+                                           atol=1e-5,
+                                           err_msg=prog_string)
 
 
 def create_backend_test(testname=None, target_device=None):
@@ -262,18 +270,15 @@ def create_backend_test(testname=None, target_device=None):
         backend_test.exclude(r'test_expand_shape_model3_cpu')
         backend_test.exclude(r'test_expand_shape_model4_cpu')
 
-        backend_test.exclude(r'test_bvlc_alexnet_cpu')
-        backend_test.exclude(r'test_densenet121_cpu')
+        # These three tests failed because of bugs in fuse_ops related to conv
+        # to be investigated later
         backend_test.exclude(r'test_inception_v1_cpu')
-        backend_test.exclude(r'test_inception_v2_cpu')
         backend_test.exclude(r'test_resnet50_cpu')
-        backend_test.exclude(r'test_shufflenet_cpu')
         backend_test.exclude(r'test_squeezenet_cpu')
-        backend_test.exclude(r'test_vgg19_cpu')
-        backend_test.exclude(r'test_zfnet512_cpu')
 
-    # import all test cases at global scope to make
-    # them visible to python.unittest.
+
+# import all test cases at global scope to make
+# them visible to python.unittest.
     globals().update(backend_test.enable_report().test_cases)
 
     return backend_test

tools/download_models.sh

+#!/bin/bash
+
+if [ -z "$ONNX_HOME" ]
+then
+   ONNX_HOME=$HOME
+fi
+
+model_dir=$ONNX_HOME/.onnx/models
+tmp_dir=$ONNX_HOME/tmp/
+mkdir -p $model_dir
+mkdir -p $tmp_dir
+models="bvlc_alexnet \
+        densenet121 \
+        inception_v2 \
+        shufflenet \
+        vgg19 \
+        zfnet512"
+
+for name in $models
+do
+curl https://s3.amazonaws.com/download.onnx/models/opset_9/$name.tar.gz --output $tmp_dir/$name.tar.gz
+tar -xzvf $tmp_dir/$name.tar.gz --directory $model_dir && rm $tmp_dir/$name.tar.gz
+done
+