Use /opt/rocm/llvm/clang-format to format (#1967)

This adds a format.py file which will check or apply formatting using git clang-format or yapf.

.githooks/pre-commit

@@ -4,62 +4,7 @@
 # are installed, and if so, uses the installed version to format
 # the staged changes.
 
-base=clang-format-10
-format=""
-yapf_base=yapf
-yapf_format=""
-
-use_yapf=true
-
-# Redirect output to stderr.
-exec 1>&2
-
-# check if clang-format is installed
-type "$base" >/dev/null 2>&1 && format="$base"
-
-# no versions of clang-format are installed
-if [ -z "$format" ]
-then
-    echo "$base is not installed. Pre-commit hook will not be executed."
-    exit 0
-fi
-
-# check if yapf is installed
-type "$yapf_base" >/dev/null 2>&1 && yapf_format="$yapf_base"
-
-# no versions of yapf are installed
-if [ -z "$yapf_format" ]
-then
-    echo "$yapf_base is not installed. Pre-commit hook for python files will not be executed"
-    use_yapf=false
-fi
-
 # Do everything from top - level
 cd $(git rev-parse --show-toplevel)
 
-if git rev-parse --verify HEAD >/dev/null 2>&1
-then
-    against=HEAD
-else
-    # Initial commit: diff against an empty tree object
-    against=16bbb57
-fi
-
-# do the formatting
-for file in $(git diff-index --cached --name-only $against | grep -E '\.h$|\.hpp$|\.cpp$|\.cl$|\.c$|\.h\.in$|\.hpp\.in$|\.cpp\.in$|\.py$')
-do
-    if [ -e "$file" ]
-    then
-        if [ $(echo $file | grep -E '\.py$') ]
-        then
-            if $use_yapf
-            then
-                echo "$yapf_format $file"
-                "$yapf_format" -i "$file"
-            fi
-        else
-            echo "$format $file"
-            "$format" -i -style=file "$file"
-        fi
-    fi
-done
+python3 tools/format.py -q -i HEAD

.github/workflows/ci.yaml

@@ -144,6 +144,8 @@ jobs:
     runs-on: ROCM-Ubuntu
     steps:
     - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
 
     # In this step, this action saves a list of existing images,
     # the cache is created without them in the post run.
@@ -164,19 +166,8 @@ jobs:
       shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
       run: |
         set -e
-        find . -iname '*.h' \
-              -o -iname '*.hpp' \
-              -o -iname '*.cpp' \
-              -o -iname '*.h.in' \
-              -o -iname '*.hpp.in' \
-              -o -iname '*.cpp.in' \
-              -o -iname '*.cl' \
-              -o -iname '*.c' \
-          | grep -v 'build/' \
-          | xargs -n 1 -P 1 -I{} -t sh -c 'clang-format-10 -style=file {} | diff - {}'
-          find . -iname '*.py' \
-          | grep -v 'build/'  \
-          | xargs -n 1 -P 1 -I{} -t sh -c 'yapf {} | diff - {}'
+        git config --global --add safe.directory /data
+        python3 tools/format.py origin/${{ github.event_name == 'pull_request' && github.base_ref || 'develop' }}
 
   pyflakes:
     runs-on: ubuntu-20.04

Dockerfile

@@ -19,7 +19,6 @@ RUN sh -c "echo 'Package: *\nPin: release o=repo.radeon.com\nPin-priority: 600'
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
     apt-utils \
     build-essential \
-    clang-format-10 \
     cmake \
     curl \
     doxygen \

hip-clang.docker

@@ -12,7 +12,6 @@ RUN sh -c 'echo deb [arch=amd64 trusted=yes] http://repo.radeon.com/rocm/apt/5.6
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
     apt-utils \
     build-essential \
-    clang-format-10 \
     cmake \
     curl \
     doxygen \

src/include/migraphx/operation.hpp

@@ -251,9 +251,10 @@ auto compute_op(rank<1>,
                 const shape& output,
                 const std::vector<argument>& inputs,
                 const std::vector<module_ref>& module_args,
-                F f)
-    -> decltype(
-        x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f))
+                F f) -> decltype(x.compute(make_compute_output_shape(pack(x, output, inputs)),
+                                           inputs,
+                                           module_args,
+                                           f))
 {
     return x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f);
 }
@@ -309,9 +310,10 @@ auto compute_op(rank<3>,
                 const shape& output,
                 const std::vector<argument>& inputs,
                 const std::vector<module_ref>& module_args,
-                F f)
-    -> decltype(
-        x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f))
+                F f) -> decltype(x.compute(make_compute_output_shape(pack(x, output, inputs)),
+                                           inputs,
+                                           module_args,
+                                           f))
 {
     return x.compute(make_compute_output_shape(pack(x, output, inputs)), inputs, module_args, f);
 }

tools/format.py

+#####################################################################################
+# The MIT License (MIT)
+#
+# Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#####################################################################################
+import os, shutil, argparse, subprocess
+
+CLANG_FORMAT_PATH = '/opt/rocm/llvm/bin'
+
+
+def run(cmd, **kwargs):
+    print(cmd)
+    subprocess.run(cmd, shell=True, check=True, **kwargs)
+
+
+def eval(cmd, **kwargs):
+    return subprocess.run(cmd,
+                          capture_output=True,
+                          shell=True,
+                          check=True,
+                          **kwargs).stdout.decode('utf-8').strip()
+
+
+def get_top():
+    return eval("git rev-parse --show-toplevel")
+
+
+def get_head():
+    return eval("git rev-parse --abbrev-ref HEAD")
+
+
+def get_merge_base(branch):
+    head = get_head()
+    return eval(f"git merge-base {branch} {head}")
+
+
+def clang_format(against, apply=False, path=CLANG_FORMAT_PATH):
+    base = get_merge_base(against)
+    clang_format = os.path.join(path, 'clang-format')
+    if not os.path.exists(clang_format):
+        print(f"{clang_format} not installed. Skipping format.")
+        return
+    git_clang_format = os.path.join(path, 'git-clang-format')
+    if not os.path.exists(git_clang_format):
+        print(f"{git_clang_format} not installed. Skipping format.")
+        return
+    diff_flag = "" if apply else "--diff"
+    run(f"{git_clang_format} --binary {clang_format} {diff_flag} {base}")
+
+
+def get_files_changed(against, ext=('py')):
+    files = eval(f"git diff-index --cached --name-only {against}",
+                 cwd=get_top()).splitlines()
+    return (f for f in files if f.endswith(ext))
+
+
+def yapf_format(against, apply=False):
+    if not shutil.which('yapf'):
+        print("yapf not installed. Skipping format.")
+        return
+    diff_flag = "--in-place" if apply else "--diff"
+    files = ' '.join(get_files_changed(against))
+    if files:
+        run(f"yapf {diff_flag} -p {files}")
+    else:
+        print("No modified python files to format")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('against', default='develop', nargs='?')
+    parser.add_argument('-i', '--in-place', action='store_true')
+    parser.add_argument('-q', '--quiet', action='store_true')
+    args = parser.parse_args()
+    try:
+        clang_format(args.against, apply=args.in_place)
+        yapf_format(args.against, apply=args.in_place)
+    except subprocess.CalledProcessError as ex:
+        if ex.stdout:
+            print(ex.stdout.decode('utf-8'))
+        if ex.stderr:
+            print(ex.stderr.decode('utf-8'))
+        if not args.quiet:
+            print(f"Command '{ex.cmd}' returned {ex.returncode}")
+            raise
+            # sys.exit(ex.returncode)
+
+
+if __name__ == "__main__":
+    main()

tools/generate.sh

@@ -22,6 +22,7 @@
 # THE SOFTWARE.
 #####################################################################################
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CLANG_FORMAT=/opt/rocm/llvm/bin/clang-format
 SRC_DIR=$DIR/../src
 PYTHON=python3
 if type -p python3.6 > /dev/null ; then
@@ -30,10 +31,10 @@ fi
 if type -p python3.8 > /dev/null ; then
     PYTHON=python3.8
 fi
-ls -1 $DIR/include/ | xargs -n 1 -P $(nproc) -I{} -t bash -c "$PYTHON $DIR/te.py $DIR/include/{} | clang-format-10 -style=file > $SRC_DIR/include/migraphx/{}"
+ls -1 $DIR/include/ | xargs -n 1 -P $(nproc) -I{} -t bash -c "$PYTHON $DIR/te.py $DIR/include/{} | $CLANG_FORMAT -style=file > $SRC_DIR/include/migraphx/{}"
 
 function api {
-    $PYTHON $DIR/api.py $SRC_DIR/api/migraphx.py $1 | clang-format-10 -style=file > $2
+    $PYTHON $DIR/api.py $SRC_DIR/api/migraphx.py $1 | $CLANG_FORMAT -style=file > $2
 }
 
 api $DIR/api/migraphx.h $SRC_DIR/api/include/migraphx/migraphx.h