Merge branch 'develop' into resnet50_partition

tools/check_stamped.py

@@ -2,7 +2,7 @@
 #####################################################################################
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+#  Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -27,11 +27,11 @@ import sys
 
 debug = False
 # The filetypes we want to check for that are stamped
-# LICENSE is included here as it SHOULD have a liscence in it otherwise flag it as unstamped
+# LICENSE is included here as it SHOULD have a license in it otherwise flag it as unstamped
 supported_file_types = (".cpp", ".hpp", ".h", ".ipynb", ".py", ".txt", ".sh",
                         ".bsh", "LICENSE", ".cmake")
 
-#add general stuff we shouldn't stamp and any exceptions here
+# add general stuff we shouldn't stamp and any exceptions here
 unsupported_file_types = [
     ".onnx", ".pb", ".rst", ".jpg", ".jpeg", ".proto", ".md", ".clang",
     ".weight", ".ini", ".json", ".docker", ".git", ".rules", ".yml"
@@ -40,105 +40,89 @@ unsupported_file_types = [
 specificIgnores = ("digits.txt", "Dockerfile", "Jenkinsfile", "")
 
 
-def hasKeySequence(inputfile, key_message):
-    result = False
+def hasKeySequence(inputfile: str, key_message: str) -> bool:
     if key_message in inputfile:
-        result = True
-    return result
+        return True
+    return False
 
 
-#Simple just open and write stuff to each file with the license stamp
-def openAndCheckFile(filename):
-    result = False
-    #open save old contents and append things here
-    if debug is True:
-        print("Open", filename, end='')
+# Simple just open and write stuff to each file with the license stamp
+def needStampCheck(filename: str) -> bool:
+    # open save old contents and append things here
+    if debug: print("Open", filename, end=' ')
 
     try:
         file = open(filename, 'r')
     except OSError as e:
-        if debug is True:
-            print(str(e) + "....Open Error: Skipping  file ")
+        if debug: print(str(e) + "....Open Error: Skipping  file ")
         file.close()
-        return
+        return False
     else:
         with file as contents:
             try:
                 save = contents.read()
-                hasAmdLic = hasKeySequence(
-                    save, "Advanced Micro Devices, Inc. All rights reserved")
 
-                #Check if we have a licence stamp already
-                if hasAmdLic is True:
-                    if debug is True:
-                        print("....Already Stamped: Skipping  file ")
+                # Check if we have a license stamp already
+                if hasKeySequence(
+                        save,
+                        "Advanced Micro Devices, Inc. All rights reserved"):
+                    if debug: print("....Already Stamped: Skipping  file ")
                     contents.close()
-                    result = True
+                    return False
 
             except UnicodeDecodeError as eu:
-                if debug is True:
-                    print(str(eu) + "...Skipping binary file ")
+                if debug: print(f"{str(eu)}...Skipping binary file ")
                 contents.close()
-                result = True
+                return False
 
-    return result
+    return True
 
 
-# Deterine if filename is desired in the fileTuple past in
-def check_filename(filename, fileTuple):
-    supported = False
-    for key in fileTuple:
-        if key in filename:
-            supported = True
-            break
-    return supported
+# Check if any element in fileTuple is in filename
+def check_filename(filename: str, fileTuple: tuple or list) -> bool:
+    if any([x in filename for x in fileTuple]):
+        return True
+    return False
 
 
-def main():
+def main() -> None:
     unsupported_file_types.extend(specificIgnores)
 
-    #Get a list of all the tracked files in our git repo
+    # Get a list of all the tracked files in our git repo
     proc = subprocess.run("git ls-files --exclude-standard",
                           shell=True,
                           stdout=subprocess.PIPE)
     fileList = proc.stdout.decode().split('\n')
 
-    if debug is True:
-        print("Target file list:\n" + str(fileList))
+    if debug: print("Target file list:\n" + str(fileList))
 
     unsupportedFiles = []
     unstampedFiles = []
     unknownFiles = []
 
     for file in fileList:
-        supported = check_filename(file, supported_file_types)
-        if supported is True:
-            isStamped = openAndCheckFile(file)
-            if isStamped is False:
+        if check_filename(file, supported_file_types):
+            if needStampCheck(file):
                 unstampedFiles.append(file)
 
+        elif check_filename(file, unsupported_file_types):
+            unsupportedFiles.append(file)
         else:
-            unsupported = check_filename(file, unsupported_file_types)
-            if unsupported is True:
-                unsupportedFiles.append(file)
-            else:
-                unknownFiles.append(file)
+            unknownFiles.append(file)
 
-    #Do a bunch of checks based on our file lists
+    # Do a bunch of checks based on our file lists
     if len(unstampedFiles) > 0:
-        print("Error: The following " + str(len(unstampedFiles)) +
+        print("\nError: The following " + str(len(unstampedFiles)) +
               " files are currently without a license:")
         print(str(unstampedFiles))
         sys.exit(1)
 
     if len(unknownFiles) > 0:
-        print("Error: The following " + str(len(unknownFiles)) +
+        print("\nError: The following " + str(len(unknownFiles)) +
               " files not handled:")
         print(str(unknownFiles))
         sys.exit(2)
 
-    sys.exit(0)
-
 
 if __name__ == "__main__":
     main()

tools/docker/sles.docker

@@ -3,7 +3,7 @@ FROM registry.suse.com/suse/sle15:15.4
 RUN sh -c 'echo -e "\
 [rocm]\n\
 name=rocm\n\
-baseurl=https://repo.radeon.com/rocm/zyp/5.6/main\n\
+baseurl=https://repo.radeon.com/rocm/zyp/5.7/main\n\
 enabled=1\n\
 gpgcheck=1\n\
 gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key\n\

tools/docker/ubuntu_2204.dockerfile

@@ -10,7 +10,7 @@ RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl &&
     curl -fsSL http://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg
 
 # Add rocm repository
-RUN sh -c "echo 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] http://repo.radeon.com/rocm/apt/5.5 jammy main' > /etc/apt/sources.list.d/rocm.list"
+RUN sh -c "echo 'deb [arch=amd64 signed-by=/etc/apt/trusted.gpg.d/rocm-keyring.gpg] http://repo.radeon.com/rocm/apt/5.7 jammy main' > /etc/apt/sources.list.d/rocm.list"
 
 # From docs.amd.com for installing rocm. Needed to install properly
 RUN sh -c "echo 'Package: *\nPin: release o=repo.radeon.com\nPin-priority: 600' > /etc/apt/preferences.d/rocm-pin-600"
@@ -87,7 +87,7 @@ RUN test -f /usr/local/hash || exit 1
 RUN pip3 install yapf==0.28.0
 
 # Install doc requirements
-ADD doc/requirements.txt /doc-requirements.txt
+ADD docs/.sphinx/requirements.txt /doc-requirements.txt
 RUN pip3 install -r /doc-requirements.txt
 
 # Download real models to run onnx unit tests

tools/download_models.sh

@@ -49,3 +49,8 @@ do
 curl https://download.onnxruntime.ai/onnx/models/$name.tar.gz --output $tmp_dir/$name.tar.gz
 tar -xzvf $tmp_dir/$name.tar.gz --directory $model_dir && rm $tmp_dir/$name.tar.gz
 done
+
+# CI jobs can run as a different user then the docker image builder.
+# Allow read/write access to the models
+chmod 777 $model_dir
+

tools/generate.sh → tools/generate.py

 #####################################################################################
 # The MIT License (MIT)
 #
-# Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -21,23 +21,68 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 #####################################################################################
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-CLANG_FORMAT=/opt/rocm/llvm/bin/clang-format
-SRC_DIR=$DIR/../src
-PYTHON=python3
-if type -p python3.6 > /dev/null ; then
-    PYTHON=python3.6
-fi
-if type -p python3.8 > /dev/null ; then
-    PYTHON=python3.8
-fi
-ls -1 $DIR/include/ | xargs -n 1 -P $(nproc) -I{} -t bash -c "$PYTHON $DIR/te.py $DIR/include/{} | $CLANG_FORMAT -style=file > $SRC_DIR/include/migraphx/{}"
-
-function api {
-    $PYTHON $DIR/api.py $SRC_DIR/api/migraphx.py $1 | $CLANG_FORMAT -style=file > $2
-}
-
-api $DIR/api/migraphx.h $SRC_DIR/api/include/migraphx/migraphx.h
-echo "Finished generating header migraphx.h"
-api $DIR/api/api.cpp $SRC_DIR/api/api.cpp
-echo "Finished generating source api.cpp "
+import api, argparse, os, runpy, subprocess, sys, te
+from pathlib import Path
+
+clang_format_path = Path('clang-format.exe' if os.name ==
+                         'nt' else '/opt/rocm/llvm/bin/clang-format')
+work_dir = Path().cwd()
+src_dir = (work_dir / '../src').absolute()
+migraphx_py_path = src_dir / 'api/migraphx.py'
+
+
+def clang_format(buffer, **kwargs):
+    return subprocess.run(f'{clang_format_path} -style=file',
+                          capture_output=True,
+                          shell=True,
+                          check=True,
+                          input=buffer.encode('utf-8'),
+                          cwd=work_dir,
+                          **kwargs).stdout.decode('utf-8')
+
+
+def api_generate(input_path: Path, output_path: Path):
+    with open(output_path, 'w') as f:
+        f.write(clang_format(api.run(input_path)))
+
+
+def te_generate(input_path: Path, output_path: Path):
+    with open(output_path, 'w') as f:
+        f.write(clang_format(te.run(input_path)))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--clang-format', type=Path)
+    args = parser.parse_args()
+
+    global clang_format_path
+    if args.clang_format:
+        clang_format_path = args.clang_format
+
+    if not clang_format_path.is_file():
+        print(f"{clang_format_path}: invalid path or not installed",
+              file=sys.stderr)
+        return
+
+    try:
+        files = Path('include').absolute().iterdir()
+        for f in [f for f in files if f.is_file()]:
+            te_generate(f, src_dir / f'include/migraphx/{f.name}')
+        runpy.run_path(str(migraphx_py_path))
+        api_generate(work_dir / 'api/migraphx.h',
+                     src_dir / 'api/include/migraphx/migraphx.h')
+        print('Finished generating header migraphx.h')
+        api_generate(work_dir / 'api/api.cpp', src_dir / 'api/api.cpp')
+        print('Finished generating source api.cpp')
+    except subprocess.CalledProcessError as ex:
+        if ex.stdout:
+            print(ex.stdout.decode('utf-8'))
+        if ex.stderr:
+            print(ex.stdout.decode('utf-8'))
+        print(f"Command '{ex.cmd}' returned {ex.returncode}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

tools/install_prereqs.sh

@@ -80,8 +80,8 @@ rbuild prepare -d $PREFIX -s develop
 if [[ ("${ID}" != "sles") ]]; then
 export CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=ON"
 
-pip3 install onnx==1.10.2 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0
+pip3 install onnx==1.14.1 numpy==1.21.6 typing==3.7.4 pytest==6.0.1 packaging==23.0
 
 # pin version of protobuf in Python for onnx runtime unit tests between dist versions
-pip3 install protobuf==3.20.0
+pip3 install protobuf==3.20.2
 fi

tools/license_stamper.py

@@ -2,7 +2,7 @@
 #####################################################################################
 #  The MIT License (MIT)
 #
-#  Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+#  Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 #  Permission is hereby granted, free of charge, to any person obtaining a copy
 #  of this software and associated documentation files (the "Software"), to deal
@@ -38,7 +38,7 @@ def getipynb_markdownBlockAsList():
         '\t\t"cell_type": "code",\n', '\t\t"execution_count": null,\n',
         '\t\t"metadata": {},\n', '\t\t"outputs": [],\n', '\t\t"source": [\n',
         '\t\t\t\"#  The MIT License (MIT)\\n\",\n', '\t\t\t\"#\\n\",\n',
-        '\t\t\t\"#  Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.\\n\",\n',
+        '\t\t\t\"#  Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.\\n\",\n',
         '\t\t\t\"#\\n\",\n',
         '\t\t\t\"#  Permission is hereby granted, free of charge, to any person obtaining a copy\\n\",\n',
         '\t\t\t\"#  of this software and associated documentation files (the \'Software\'), to deal\\n\",\n',

tools/te.py

@@ -431,6 +431,9 @@ def template_eval(template, **kwargs):
     return template
 
 
-f = open(sys.argv[1]).read()
-r = template_eval(f)
-sys.stdout.write(r)
+def run(p):
+    return template_eval(open(p).read())
+
+
+if __name__ == '__main__':
+    sys.stdout.write(run(sys.argv[1]))