Yapf hook (#369)

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

* test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

* # This is a combination of 3 commits.
# The first commit's message is:
# This is a combination of 2 commits.
# The first commit's message is:

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

test hook

# This is the 2nd commit message:

test hook

# This is the 2nd commit message:

test hook

# This is the 3rd commit message:

test hook

* fix format

* fix format

* ignore doc dir

* fix regex

* fix jenkins error

* exclude another dir

* formatting test_array

* fix version of yapf

* test hook

* formatting

* reinclude dirs

.githooks/pre-commit

@@ -6,11 +6,15 @@
 
 base=clang-format-5.0
 format=""
+yapf_base=yapf
+yapf_format=""
+
+use_yapf=true
 
 # Redirect output to stderr.
 exec 1>&2
 
- # check if clang-format is installed
+# check if clang-format is installed
 type "$base" >/dev/null 2>&1 && format="$base"
 
 # no versions of clang-format are installed
@@ -20,6 +24,16 @@ then
     exit 0
 fi
 
+# check if yapf is installed
+type "$yapf_base" >/dev/null 2>&1 && yapf_format="$yapf_base"
+
+# no versions of yapf are installed
+if [ -z "$yapf_format" ]
+then
+    echo "$yapf_base is not installed. Pre-commit hook for python files will not be executed"
+    use_yapf=false
+fi
+
 # Do everything from top - level
 cd $(git rev-parse --show-toplevel)
 
@@ -32,12 +46,20 @@ else
 fi
 
 # do the formatting
-for file in $(git diff-index --cached --name-only $against | grep -E '\.h$|\.hpp$|\.cpp$|\.cl$|\.h\.in$|\.hpp\.in$|\.cpp\.in$')
+for file in $(git diff-index --cached --name-only $against | grep -E '\.h$|\.hpp$|\.cpp$|\.cl$|\.h\.in$|\.hpp\.in$|\.cpp\.in$|\.py$')
 do
     if [ -e "$file" ]
     then
-        echo "$format $file"
-        "$format" -i -style=file "$file"
+        if [ $(echo $file | grep -E '\.py$') ]
+        then
+            if $use_yapf
+            then
+                echo "$yapf_format $file"
+                "$yapf_format" -i "$file"
+            fi
+        else
+            echo "$format $file"
+            "$format" -i -style=file "$file"
+        fi
     fi
 done
-

Dockerfile

@@ -52,6 +52,9 @@ RUN pip install cget
 # Install rclone
 RUN pip install https://github.com/pfultz2/rclone/archive/master.tar.gz
 
+# Install yapf
+RUN pip install yapf==0.28.0
+
 # Install hcc
 RUN rclone -b roc-2.6.x -c 0f4c96b7851af2663a7f3ac16ecfb76c7c78a5bf https://github.com/RadeonOpenCompute/hcc.git /hcc
 RUN cget -p $PREFIX install hcc,/hcc

Jenkinsfile

@@ -86,7 +86,7 @@ rocmtest tidy: rocmnode('rocmtest') { cmake_build ->
         '''
     }
 }, format: rocmnode('rocmtest') { cmake_build ->
-    stage('Clang Format') {
+    stage('Format') {
         sh '''
             find . -iname \'*.h\' \
                 -o -iname \'*.hpp\' \
@@ -97,6 +97,9 @@ rocmtest tidy: rocmnode('rocmtest') { cmake_build ->
                 -o -iname \'*.cl\' \
             | grep -v 'build/' \
             | xargs -n 1 -P 1 -I{} -t sh -c \'clang-format-5.0 -style=file {} | diff - {}\'
+            find . -iname \'*.py\' \
+            | grep -v 'build/'  \
+            | xargs -n 1 -P 1 -I{} -t sh -c \'yapf {} | diff - {}\'
         '''
     }
 }, clang_debug: rocmnode('vega') { cmake_build ->

doc/src/conf.py

@@ -20,7 +20,6 @@
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
 
-
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -78,7 +77,6 @@ highlight_language = 'cpp'
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
-
 # -- Options for HTML output ----------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -97,13 +95,11 @@ html_theme = 'alabaster'
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
 
-
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'MIGraphXdoc'
 
-
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
@@ -128,20 +124,14 @@ latex_elements = {
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'MIGraphX.tex', u'MIGraphX Documentation',
-     u'AMD', 'manual'),
+    (master_doc, 'MIGraphX.tex', u'MIGraphX Documentation', u'AMD', 'manual'),
 ]
 
-
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'migraphx', u'MIGraphX Documentation',
-     [author], 1)
-]
-
+man_pages = [(master_doc, 'migraphx', u'MIGraphX Documentation', [author], 1)]
 
 # -- Options for Texinfo output -------------------------------------------
 
@@ -149,9 +139,8 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'MIGraphX', u'MIGraphX Documentation',
-     author, 'MIGraphX', 'One line description of project.',
-     'Miscellaneous'),
+    (master_doc, 'MIGraphX', u'MIGraphX Documentation', author, 'MIGraphX',
+     'One line description of project.', 'Miscellaneous'),
 ]
 
 breathe_default_members = ('members', 'undoc-members')
@@ -159,4 +148,3 @@ cpp_index_common_prefix = ['migraphx::']
 
 default_role = 'any'
 primary_domain = 'cpp'
-

test/onnx/gen_onnx.py

@@ -153,6 +153,7 @@ def cast_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def ceil_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [10])
@@ -166,6 +167,7 @@ def ceil_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def clip_test():
     x = helper.make_tensor_value_info('0', TensorProto.FLOAT, [3])
@@ -647,6 +649,7 @@ def flatten_test():
 
     return ([node, node2], [x], [y, y2])
 
+
 @onnx_test
 def floor_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [10])
@@ -660,6 +663,7 @@ def floor_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def gather_test():
     x = helper.make_tensor_value_info('data', TensorProto.FLOAT, [3, 4, 5, 6])
@@ -1053,6 +1057,7 @@ def pow_test():
 
     return ([node], [arg0, arg1], [arg_out])
 
+
 @onnx_test
 def reducemax_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [3, 4, 5, 6])
@@ -1082,6 +1087,7 @@ def reducemean_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def reducemean_keepdims_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [3, 4, 5, 6])
@@ -1096,6 +1102,7 @@ def reducemean_keepdims_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def reducemin_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [3, 4, 5, 6])
@@ -1110,6 +1117,7 @@ def reducemin_test():
 
     return ([node], [x], [y])
 
+
 @onnx_test
 def reducesum_test():
     x = helper.make_tensor_value_info('x', TensorProto.FLOAT, [3, 4, 5, 6])

test/py/test_array.py

@@ -4,20 +4,25 @@ try:
 except:
     pass
 
+
 def assert_eq(x, y):
     if x == y:
         pass
     else:
         raise Exception(str(x) + " != " + str(y))
 
+
 def read_float(b, index):
-    return struct.unpack_from('f', b, index*4)[0]
+    return struct.unpack_from('f', b, index * 4)[0]
+
 
 def write_float(b, index):
-    struct.pack_into('f', b, index*4)
+    struct.pack_into('f', b, index * 4)
+
 
 def nelements(lens):
-    return reduce(lambda x,y: x*y,lens, 1)
+    return reduce(lambda x, y: x * y, lens, 1)
+
 
 def create_buffer(t, data, shape):
     a = array.array(t, data)
@@ -28,19 +33,22 @@ def create_buffer(t, data, shape):
         m = memoryview(a.tostring())
         return m
 
+
 def check_argument(a):
     l = a.tolist()
     for i in range(len(l)):
         assert_eq(l[i], read_float(a, i))
 
+
 def check_shapes(r, m):
     lens = list(m.shape)
-    strides = [int(s/m.itemsize) for s in m.strides]
+    strides = [int(s / m.itemsize) for s in m.strides]
     elements = nelements(lens)
     assert_eq(r.get_shape().elements(), elements)
     assert_eq(r.get_shape().lens(), lens)
     assert_eq(r.get_shape().strides(), strides)
 
+
 def run(p):
     params = {}
     for key, value in p.get_parameter_shapes().items():
@@ -48,6 +56,7 @@ def run(p):
 
     return migraphx.from_gpu(p.run(params))
 
+
 def test_shape(shape):
     data = list(range(nelements(shape)))
     m = create_buffer('f', data, shape)
@@ -55,6 +64,7 @@ def test_shape(shape):
     check_shapes(a, m)
     assert_eq(a.tolist(), data)
 
+
 def test_input():
     if sys.version_info >= (3, 0):
         test_shape([4])

test/py/test_cpu.py

@@ -9,6 +9,7 @@ print(p)
 s2 = p.get_shape()
 assert s1 == s2
 params = {}
+
 for key, value in p.get_parameter_shapes().items():
     print("Parameter {} -> {}".format(key, value))
     params[key] = migraphx.generate_argument(value)

test/py/test_gpu.py

@@ -6,6 +6,7 @@ print("Compiling ...")
 p.compile(migraphx.get_target("gpu"))
 print(p)
 params = {}
+
 for key, value in p.get_parameter_shapes().items():
     print("Parameter {} -> {}".format(key, value))
     params[key] = migraphx.to_gpu(migraphx.generate_argument(value))

tools/te.py

 import string, sys, re, os
 
-
-trivial = [
-    'std::size_t',
-    'instruction_ref'
-]
+trivial = ['std::size_t', 'instruction_ref']
 
 headers = '''
 #include <algorithm>
@@ -194,7 +190,9 @@ ${friend} ${return_type} ${name}(${params}) ${const}
 }
 ''')
 
-pure_virtual_member = string.Template("virtual ${return_type} ${internal_name}(${member_params}) ${member_const} = 0;\n")
+pure_virtual_member = string.Template(
+    "virtual ${return_type} ${internal_name}(${member_params}) ${member_const} = 0;\n"
+)
 
 virtual_member = string.Template('''
 ${return_type} ${internal_name}(${member_params}) ${member_const} override
@@ -204,7 +202,9 @@ ${return_type} ${internal_name}(${member_params}) ${member_const} override
 }
 ''')
 
-comment_member = string.Template('''*     ${friend} ${return_type} ${name}(${params}) ${const};''')
+comment_member = string.Template(
+    '''*     ${friend} ${return_type} ${name}(${params}) ${const};''')
+
 
 def trim_type_name(name):
     n = name.strip()
@@ -214,6 +214,7 @@ def trim_type_name(name):
         return trim_type_name(n[0:-1])
     return n
 
+
 def internal_name(name):
     internal_names = {
         'operator<<': 'operator_shift_left',
@@ -224,6 +225,7 @@ def internal_name(name):
     else:
         return name
 
+
 def generate_call(m, friend, indirect):
     if m['name'].startswith('operator'):
         op = m['name'][8:]
@@ -236,14 +238,18 @@ def generate_call(m, friend, indirect):
         return string.Template('${name}(${args})').substitute(m)
     if indirect:
         if m['args']:
-            return string.Template('${default}(private_detail_te_value, ${args})').substitute(m)
+            return string.Template(
+                '${default}(private_detail_te_value, ${args})').substitute(m)
         else:
-            return string.Template('${default}(private_detail_te_value)').substitute(m)
-    return string.Template('private_detail_te_value.${name}(${args})').substitute(m)
+            return string.Template(
+                '${default}(private_detail_te_value)').substitute(m)
+    return string.Template(
+        'private_detail_te_value.${name}(${args})').substitute(m)
+
 
 def convert_member(d, struct_name):
     for name in d:
-        member = { 
+        member = {
             'name': name,
             'internal_name': internal_name(name),
             'const': '',
@@ -270,7 +276,8 @@ def convert_member(d, struct_name):
             t = d[name][x]
             if x == 'return':
                 member['return_type'] = t if t else 'void'
-                if member['return_type'] != 'void': member['return_'] = 'return'
+                if member['return_type'] != 'void':
+                    member['return_'] = 'return'
             elif x == 'const':
                 member['const'] = 'const'
                 member['member_const'] = 'const'
@@ -285,9 +292,9 @@ def convert_member(d, struct_name):
             elif x.startswith('__') and x.endswith('__'):
                 continue
             else:
-                use_member = not(skip and struct_name == trim_type_name(t))
+                use_member = not (skip and struct_name == trim_type_name(t))
                 arg_name = x
-                if not use_member: 
+                if not use_member:
                     arg_name = 'private_detail_te_value'
                     member['this'] = x
                     if 'const' in t:
@@ -296,10 +303,11 @@ def convert_member(d, struct_name):
                     if use_member: member_args.append(x)
                     args.append(arg_name)
                 else:
-                    if use_member: member_args.append('std::move({})'.format(x))
+                    if use_member:
+                        member_args.append('std::move({})'.format(x))
                     args.append('std::move({})'.format(arg_name))
-                params.append(t+' '+x)
-                if use_member: member_params.append(t+' '+x)
+                params.append(t + ' ' + x)
+                if use_member: member_params.append(t + ' ' + x)
                 else: skip = False
         member['args'] = ','.join(args)
         member['member_args'] = ','.join(member_args)
@@ -322,40 +330,43 @@ def generate_form(name, members):
         pure_virtual_members.append(pure_virtual_member.substitute(m))
         virtual_members.append(virtual_member.substitute(m))
         comment_members.append(comment_member.substitute(m))
-    return form.substitute(
-        nonvirtual_members=''.join(nonvirtual_members),
-        pure_virtual_members=''.join(pure_virtual_members),
-        virtual_members=''.join(virtual_members),
-        comment_members='\n'.join(comment_members),
-        struct_name=name
-    )
+    return form.substitute(nonvirtual_members=''.join(nonvirtual_members),
+                           pure_virtual_members=''.join(pure_virtual_members),
+                           virtual_members=''.join(virtual_members),
+                           comment_members='\n'.join(comment_members),
+                           struct_name=name)
+
 
 def virtual(name, returns=None, **kwargs):
     args = kwargs
     args['return'] = returns
-    return { name: args }
+    return {name: args}
+
 
 def friend(name, returns=None, **kwargs):
     args = kwargs
     args['return'] = returns
     args['friend'] = 'friend'
-    return { name: args }
+    return {name: args}
 
 
 def interface(name, *members):
     return generate_form(name, members)
 
-def template_eval(template,**kwargs):
+
+def template_eval(template, **kwargs):
     start = '<%'
     end = '%>'
     escaped = (re.escape(start), re.escape(end))
     mark = re.compile('%s(.*?)%s' % escaped, re.DOTALL)
     for key in kwargs:
-        exec('%s = %s' % (key, kwargs[key]))
+        exec ('%s = %s' % (key, kwargs[key]))
     for item in mark.findall(template):
-        template = template.replace(start+item+end, str(eval(item.strip())))
+        template = template.replace(start + item + end,
+                                    str(eval(item.strip())))
     return template
 
+
 f = open(sys.argv[1]).read()
 r = template_eval(f)
 sys.stdout.write(r)