Skip to content

GitLab

Commit 5b2b7489 authored by Alan Turner's avatar Alan Turner
Browse files

Merge remote-tracking branch 'origin/develop' into ck-flash-attn

parents 9861856e a50cb302
Hide whitespace changes
Inline Side-by-side
Showing with 254 additions and 83 deletions
src/targets/gpu/mlir.cpp
View file @ 5b2b7489
......@@ -320,7 +320,10 @@ struct mlir_program
MlirType make_tensor(const shape& s) const
{
assert(s.standard());
if(not s.standard())
MIGRAPHX_THROW("MLIR expects all tensors to be in standard shape");
if(s.dynamic())
MIGRAPHX_THROW("MLIR does not support dynamic shapes");
std::vector<int64_t> lens(s.lens().begin(), s.lens().end());
return mlirRankedTensorTypeGet(
lens.size(), lens.data(), make_type(s.type()), mlirAttributeGetNull());
......
src/tf/parse_reshape.cpp
View file @ 5b2b7489
......@@ -45,8 +45,7 @@ struct parse_reshape : op_parser<parse_reshape>
auto s = args[1]->eval();
std::vector<int64_t> dims;
s.visit([&](auto v) { copy(v, std::back_inserter(dims)); });
return info.add_instruction(make_op("reshape", {{"dims", dims}}),
info.make_contiguous(args[0]));
return info.add_instruction(make_op("reshape", {{"dims", dims}}), args[0]);
}
};
......
test/include/test.hpp
View file @ 5b2b7489
......@@ -339,6 +339,8 @@ inline std::ostream& operator<<(std::ostream& os, const color& c)
static const bool use_color = isatty(STDOUT_FILENO) != 0;
if(use_color)
return os << "\033[" << static_cast<std::size_t>(c) << "m";
#else
(void)c;
#endif
return os;
}
......
test/msgpack.cpp
View file @ 5b2b7489
......@@ -97,9 +97,12 @@ TEST_CASE(test_msgpack_bool)
TEST_CASE(test_msgpack_float)
{
migraphx::value v = 3.0;
// changed all double values in this code to not end with .0 because on msgpack for Windows if
// input type is double and ends with .0 it could be converted to uint64_t or int64_t and the
// goal of these functions is to test double without conversions
migraphx::value v = 3.01;
auto buffer = migraphx::to_msgpack(v);
EXPECT(buffer == msgpack_buffer(3.0));
EXPECT(buffer == msgpack_buffer(3.01));
EXPECT(migraphx::from_msgpack(buffer) == v);
}
......@@ -129,10 +132,10 @@ TEST_CASE(test_msgpack_empty_array)
TEST_CASE(test_msgpack_object)
{
migraphx::value v = {{"one", 1.0}, {"three", 3.0}, {"two", 2.0}};
migraphx::value v = {{"one", 1.01}, {"three", 3.01}, {"two", 2.01}};
auto buffer = migraphx::to_msgpack(v);
EXPECT(buffer == msgpack_buffer(std::map<std::string, double>{
{"one", 1.0}, {"three", 3.0}, {"two", 2.0}}));
{"one", 1.01}, {"three", 3.01}, {"two", 2.01}}));
EXPECT(migraphx::from_msgpack(buffer) == v);
}
......@@ -157,17 +160,17 @@ struct foo
TEST_CASE(test_msgpack_object_class)
{
migraphx::value v = {{"a", 1.0}, {"b", "abc"}};
migraphx::value v = {{"a", 1.01}, {"b", "abc"}};
auto buffer = migraphx::to_msgpack(v);
EXPECT(buffer == msgpack_buffer(foo{1.0, "abc"}));
EXPECT(buffer == msgpack_buffer(foo{1.01, "abc"}));
EXPECT(migraphx::from_msgpack(buffer) == v);
}
TEST_CASE(test_msgpack_array_class)
{
migraphx::value v = {{{"a", 1.0}, {"b", "abc"}}, {{"a", 3.0}, {"b", "xyz"}}};
migraphx::value v = {{{"a", 1.01}, {"b", "abc"}}, {{"a", 3.01}, {"b", "xyz"}}};
auto buffer = migraphx::to_msgpack(v);
EXPECT(buffer == msgpack_buffer(std::vector<foo>{foo{1.0, "abc"}, foo{3.0, "xyz"}}));
EXPECT(buffer == msgpack_buffer(std::vector<foo>{foo{1.01, "abc"}, foo{3.01, "xyz"}}));
EXPECT(migraphx::from_msgpack(buffer) == v);
}
......
test/onnx/gen_onnx.py
View file @ 5b2b7489
......@@ -5096,6 +5096,61 @@ def prelu_brcst_test():
return ([node], [arg0, arg1], [arg_out])
@onnx_test()
def qlinearadd_test():
a = helper.make_tensor_value_info('A', TensorProto.UINT8, [64])
sc_a = helper.make_tensor('A_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_a = helper.make_tensor('A_zero_point', TensorProto.UINT8, [], [0])
b = helper.make_tensor_value_info('B', TensorProto.UINT8, [64])
sc_b = helper.make_tensor('B_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_b = helper.make_tensor('B_zero_point', TensorProto.UINT8, [],
[128])
sc_c = helper.make_tensor('C_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_c = helper.make_tensor('C_zero_point', TensorProto.UINT8, [], [64])
c = helper.make_tensor_value_info('C', TensorProto.UINT8, [64])
node = onnx.helper.make_node(
'QLinearAdd',
inputs=[
'A', 'A_scale', 'A_zero_point', 'B', 'B_scale', 'B_zero_point',
'C_scale', 'C_zero_point'
],
outputs=['C'],
)
return ([node], [a, b], [c],
[sc_a, zero_pt_a, sc_b, zero_pt_b, sc_c, zero_pt_c])
@onnx_test()
def qlinearadd_bcast_test():
a = helper.make_tensor_value_info('A', TensorProto.INT8, [64])
sc_a = helper.make_tensor('A_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_a = helper.make_tensor('A_zero_point', TensorProto.INT8, [], [0])
b = helper.make_tensor_value_info('B', TensorProto.INT8, [1, 1, 64])
sc_b = helper.make_tensor('B_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_b = helper.make_tensor('B_zero_point', TensorProto.INT8, [], [32])
sc_c = helper.make_tensor('C_scale', TensorProto.FLOAT, [], [0.05])
zero_pt_c = helper.make_tensor('C_zero_point', TensorProto.INT8, [], [-64])
c = helper.make_tensor_value_info('C', TensorProto.INT8, [1, 1, 64])
node = onnx.helper.make_node(
'QLinearAdd',
inputs=[
'A', 'A_scale', 'A_zero_point', 'B', 'B_scale', 'B_zero_point',
'C_scale', 'C_zero_point'
],
outputs=['C'],
)
return ([node], [a, b], [c],
[sc_a, zero_pt_a, sc_b, zero_pt_b, sc_c, zero_pt_c])
@onnx_test()
def quantizelinear_test():
arg0 = helper.make_tensor_value_info('0', TensorProto.FLOAT, [5])
......
test/onnx/onnx_test.cpp
View file @ 5b2b7489
......@@ -1772,8 +1772,7 @@ TEST_CASE(depthtospace_test)
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 2, 2, 5, 5}}}), l0);
auto tmp2 = mm->add_instruction(
migraphx::make_op("transpose", {{"permutation", {0, 3, 4, 1, 5, 2}}}), tmp1);
auto tmp3 = mm->add_instruction(migraphx::make_op("contiguous"), tmp2);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 10, 10}}}), tmp3);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 10, 10}}}), tmp2);
auto prog = optimize_onnx("depthtospace_test.onnx");
EXPECT(p == prog);
}
......@@ -1787,8 +1786,7 @@ TEST_CASE(depthtospace_crd_test)
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 2, 2, 5, 5}}}), l0);
auto tmp2 = mm->add_instruction(
migraphx::make_op("transpose", {{"permutation", {0, 1, 4, 2, 5, 3}}}), tmp1);
auto tmp3 = mm->add_instruction(migraphx::make_op("contiguous"), tmp2);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 10, 10}}}), tmp3);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 10, 10}}}), tmp2);
auto prog = optimize_onnx("depthtospace_crd_test.onnx");
EXPECT(p == prog);
}
......@@ -1802,8 +1800,7 @@ TEST_CASE(depthtospace_simple_test)
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 2, 2, 2, 2, 3}}}), l0);
auto tmp2 = mm->add_instruction(
migraphx::make_op("transpose", {{"permutation", {0, 3, 4, 1, 5, 2}}}), tmp1);
auto tmp3 = mm->add_instruction(migraphx::make_op("contiguous"), tmp2);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 2, 4, 6}}}), tmp3);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 2, 4, 6}}}), tmp2);
auto prog = optimize_onnx("depthtospace_simple_test.onnx");
EXPECT(p == prog);
}
......@@ -1817,8 +1814,7 @@ TEST_CASE(spacetodepth_test)
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 2, 5, 2, 5, 2}}}), l0);
auto tmp2 = mm->add_instruction(
migraphx::make_op("transpose", {{"permutation", {0, 3, 5, 1, 2, 4}}}), tmp1);
auto tmp3 = mm->add_instruction(migraphx::make_op("contiguous"), tmp2);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 8, 5, 5}}}), tmp3);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {2, 8, 5, 5}}}), tmp2);
auto prog = optimize_onnx("spacetodepth_test.onnx");
EXPECT(p == prog);
}
......@@ -1832,8 +1828,7 @@ TEST_CASE(spacetodepth_simple_test)
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 2, 2, 2, 3, 2}}}), l0);
auto tmp2 = mm->add_instruction(
migraphx::make_op("transpose", {{"permutation", {0, 3, 5, 1, 2, 4}}}), tmp1);
auto tmp3 = mm->add_instruction(migraphx::make_op("contiguous"), tmp2);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 8, 2, 3}}}), tmp3);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 8, 2, 3}}}), tmp2);
auto prog = optimize_onnx("spacetodepth_simple_test.onnx");
EXPECT(p == prog);
}
......@@ -4856,6 +4851,59 @@ TEST_CASE(prelu_brcst_test)
EXPECT(p == prog);
}
TEST_CASE(qlinearadd_test)
{
migraphx::program p;
auto* mm = p.get_main_module();
auto a = mm->add_parameter("A", {migraphx::shape::uint8_type, {64}});
auto b = mm->add_parameter("B", {migraphx::shape::uint8_type, {64}});
auto sc_a = mm->add_literal(migraphx::literal{migraphx::shape::float_type, {0.05}});
auto z_pt_a = mm->add_literal(migraphx::literal{migraphx::shape::uint8_type, {0}});
auto sc_b = mm->add_literal(migraphx::literal{migraphx::shape::float_type, {0.05}});
auto z_pt_b = mm->add_literal(migraphx::literal{migraphx::shape::uint8_type, {128}});
auto sc_c = mm->add_literal(migraphx::literal{migraphx::shape::float_type, {0.05}});
auto z_pt_c = mm->add_literal(migraphx::literal{migraphx::shape::uint8_type, {64}});
auto scale_a_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), sc_a);
auto z_pt_a_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), z_pt_a);
auto fp_a =
mm->add_instruction(migraphx::make_op("dequantizelinear"), a, scale_a_bcast, z_pt_a_bcast);
auto scale_b_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), sc_b);
auto z_pt_b_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), z_pt_b);
auto fp_b =
mm->add_instruction(migraphx::make_op("dequantizelinear"), b, scale_b_bcast, z_pt_b_bcast);
auto fp_c = mm->add_instruction(migraphx::make_op("add"), fp_a, fp_b);
auto scale_c_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), sc_c);
auto z_pt_c_bcast =
mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", {64}}}), z_pt_c);
auto c =
mm->add_instruction(migraphx::make_op("quantizelinear"), fp_c, scale_c_bcast, z_pt_c_bcast);
mm->add_return({c});
auto prog = migraphx::parse_onnx("qlinearadd_test.onnx");
EXPECT(p.sort() == prog.sort());
}
TEST_CASE(quantizelinear_test)
{
migraphx::program p;
......@@ -5438,12 +5486,9 @@ TEST_CASE(reshape_test)
migraphx::literal{migraphx::shape{migraphx::shape::int64_type, {2}}, reshape_dims});
auto l0 = mm->add_parameter("0", migraphx::shape{migraphx::shape::float_type, {4, 2, 3}});
op.dims = reshape_dims;
auto c0 = mm->add_instruction(migraphx::make_op("contiguous"), l0);
mm->add_instruction(op, c0);
auto c1 = mm->add_instruction(migraphx::make_op("contiguous"), l0);
mm->add_instruction(op, c1);
mm->add_instruction(op, l0);
mm->add_instruction(op, l0);
auto prog = optimize_onnx("reshape_test.onnx");
EXPECT(p == prog);
}
......@@ -5456,8 +5501,7 @@ TEST_CASE(reshape_non_standard_test)
auto x = mm->add_parameter("x", s);
auto tran_x =
mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 2, 1}}}), x);
auto cont_x = mm->add_instruction(migraphx::make_op("contiguous"), tran_x);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {4, 3, 2}}}), cont_x);
mm->add_instruction(migraphx::make_op("reshape", {{"dims", {4, 3, 2}}}), tran_x);
auto prog = optimize_onnx("reshape_non_standard_test.onnx");
EXPECT(p == prog);
......
test/onnx/verify_onnx.cpp
View file @ 5b2b7489
......@@ -1245,6 +1245,79 @@ TEST_CASE(nonzero_test)
EXPECT(migraphx::verify::verify_rms_range(result_vector, gold));
}
TEST_CASE(qlinearadd_test)
{
// github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.QLinearAdd
migraphx::program p = migraphx::parse_onnx("qlinearadd_test.onnx");
p.compile(migraphx::make_target("ref"));
migraphx::shape a{migraphx::shape::uint8_type, {64}};
std::vector<uint8_t> data_a = {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24,
26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50,
52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76,
78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102,
104, 106, 108, 110, 112, 114, 116, 118, 120, 122, 124, 126};
migraphx::shape b{migraphx::shape::uint8_type, {64}};
std::vector<uint8_t> data_b = {128, 126, 124, 122, 120, 118, 116, 114, 112, 110, 108, 106, 104,
102, 100, 98, 96, 94, 92, 90, 88, 86, 84, 82, 80, 78,
76, 74, 72, 70, 68, 66, 64, 62, 60, 58, 56, 54, 52,
50, 48, 46, 44, 42, 40, 38, 36, 34, 32, 30, 28, 26,
24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2};
migraphx::parameter_map pp;
pp["A"] = migraphx::argument(a, data_a.data());
pp["B"] = migraphx::argument(b, data_b.data());
auto result = p.eval(pp).back();
std::vector<unsigned char> result_vector;
result.visit([&](auto output) { result_vector.assign(output.begin(), output.end()); });
std::vector<uint8_t> gold = {64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64,
64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64, 64};
EXPECT(migraphx::verify::verify_rms_range(result_vector, gold));
}
TEST_CASE(qlinearadd_bcast_test)
{
// github.com/microsoft/onnxruntime/blob/main/docs/ContribOperators.md#com.microsoft.QLinearAdd
migraphx::program p = migraphx::parse_onnx("qlinearadd_bcast_test.onnx");
p.compile(migraphx::make_target("ref"));
migraphx::shape a{migraphx::shape::int8_type, {64}};
std::vector<int8_t> data_a = {-64, -62, -60, -58, -56, -54, -52, -50, -48, -46, -44, -42, -40,
-38, -36, -34, -32, -30, -28, -26, -24, -22, -20, -18, -16, -14,
-12, -10, -8, -6, -4, -2, 0, 2, 4, 6, 8, 10, 12,
14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38,
40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62};
migraphx::shape b{migraphx::shape::int8_type, {1, 1, 64}};
std::vector<int8_t> data_b = {96, 94, 92, 90, 88, 86, 84, 82, 80, 78, 76, 74, 72,
70, 68, 66, 64, 62, 60, 58, 56, 54, 52, 50, 48, 46,
44, 42, 40, 38, 36, 34, 32, 30, 28, 26, 24, 22, 20,
18, 16, 14, 12, 10, 8, 6, 4, 2, 0, -2, -4, -6,
-8, -10, -12, -14, -16, -18, -20, -22, -24, -26, -28, -30};
migraphx::parameter_map pp;
pp["A"] = migraphx::argument(a, data_a.data());
pp["B"] = migraphx::argument(b, data_b.data());
auto result = p.eval(pp).back();
std::vector<int8_t> result_vector;
result.visit([&](auto output) { result_vector.assign(output.begin(), output.end()); });
std::vector<int8_t> gold = {-64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64,
-64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64,
-64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64,
-64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64,
-64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64, -64};
EXPECT(migraphx::verify::verify_rms_range(result_vector, gold));
}
TEST_CASE(resize_downsample_f_test)
{
migraphx::program p = migraphx::parse_onnx("resize_downsample_f_test.onnx");
......
test/verify/run_verify.cpp
View file @ 5b2b7489
......@@ -44,8 +44,7 @@ MIGRAPHX_DECLARE_ENV_VAR(MIGRAPHX_DUMP_TEST)
// An improved async, that doesn't block
template <class Function>
std::future<typename std::result_of<Function()>::type> detach_async(Function&& f,
bool parallel = true)
std::future<std::invoke_result_t<Function>> detach_async(Function&& f, bool parallel = true)
{
if(parallel)
{
......
tools/api/api.cpp
View file @ 5b2b7489
......@@ -38,26 +38,32 @@
#include <migraphx/register_op.hpp>
#include <migraphx/json.hpp>
#include <migraphx/convert_to_json.hpp>
#include <array>
#include <algorithm>
#include <cstdarg>
namespace migraphx {
#ifdef MIGRAPHX_BUILD_TESTING
static thread_local bool disable_exception_catch = false; // NOLINT
extern "C" MIGRAPHX_C_EXPORT void migraphx_test_private_disable_exception_catch(bool b)
{
disable_exception_catch = b;
}
#endif
template <class F>
migraphx_status try_(F f, bool output = true) // NOLINT
{
#ifdef MIGRAPHX_BUILD_TESTING
if(disable_exception_catch)
{
f();
}
else
{
#endif
try
{
f();
......@@ -81,7 +87,9 @@ migraphx_status try_(F f, bool output = true) // NOLINT
{
return migraphx_status_unknown_error;
}
#ifdef MIGRAPHX_BUILD_TESTING
}
#endif
return migraphx_status_success;
}
......
tools/api/migraphx.h
View file @ 5b2b7489
......@@ -26,6 +26,7 @@
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>
#include <migraphx/api/export.h>
......
tools/check_stamped.py
View file @ 5b2b7489
......@@ -2,7 +2,7 @@
#####################################################################################
# The MIT License (MIT)
#
# Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
# Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
......@@ -27,11 +27,11 @@ import sys
debug = False
# The filetypes we want to check for that are stamped
# LICENSE is included here as it SHOULD have a liscence in it otherwise flag it as unstamped
# LICENSE is included here as it SHOULD have a license in it otherwise flag it as unstamped
supported_file_types = (".cpp", ".hpp", ".h", ".ipynb", ".py", ".txt", ".sh",
".bsh", "LICENSE", ".cmake")
#add general stuff we shouldn't stamp and any exceptions here
# add general stuff we shouldn't stamp and any exceptions here
unsupported_file_types = [
".onnx", ".pb", ".rst", ".jpg", ".jpeg", ".proto", ".md", ".clang",
".weight", ".ini", ".json", ".docker", ".git", ".rules", ".yml"
......@@ -40,105 +40,89 @@ unsupported_file_types = [
specificIgnores = ("digits.txt", "Dockerfile", "Jenkinsfile", "")
def hasKeySequence(inputfile, key_message):
result = False
def hasKeySequence(inputfile: str, key_message: str) -> bool:
if key_message in inputfile:
result = True
return result
return True
return False
#Simple just open and write stuff to each file with the license stamp
def openAndCheckFile(filename):
result = False
#open save old contents and append things here
if debug is True:
print("Open", filename, end='')
# Simple just open and write stuff to each file with the license stamp
def needStampCheck(filename: str) -> bool:
# open save old contents and append things here
if debug: print("Open", filename, end=' ')
try:
file = open(filename, 'r')
except OSError as e:
if debug is True:
print(str(e) + "....Open Error: Skipping file ")
if debug: print(str(e) + "....Open Error: Skipping file ")
file.close()
return
return False
else:
with file as contents:
try:
save = contents.read()
hasAmdLic = hasKeySequence(
save, "Advanced Micro Devices, Inc. All rights reserved")
#Check if we have a licence stamp already
if hasAmdLic is True:
if debug is True:
print("....Already Stamped: Skipping file ")
# Check if we have a license stamp already
if hasKeySequence(
save,
"Advanced Micro Devices, Inc. All rights reserved"):
if debug: print("....Already Stamped: Skipping file ")
contents.close()
result = True
return False
except UnicodeDecodeError as eu:
if debug is True:
print(str(eu) + "...Skipping binary file ")
if debug: print(f"{str(eu)}...Skipping binary file ")
contents.close()
result = True
return False
return result
return True
# Deterine if filename is desired in the fileTuple past in
def check_filename(filename, fileTuple):
supported = False
for key in fileTuple:
if key in filename:
supported = True
break
return supported
# Check if any element in fileTuple is in filename
def check_filename(filename: str, fileTuple: tuple or list) -> bool:
if any([x in filename for x in fileTuple]):
return True
return False
def main():
def main() -> None:
unsupported_file_types.extend(specificIgnores)
#Get a list of all the tracked files in our git repo
# Get a list of all the tracked files in our git repo
proc = subprocess.run("git ls-files --exclude-standard",
shell=True,
stdout=subprocess.PIPE)
fileList = proc.stdout.decode().split('\n')
if debug is True:
print("Target file list:\n" + str(fileList))
if debug: print("Target file list:\n" + str(fileList))
unsupportedFiles = []
unstampedFiles = []
unknownFiles = []
for file in fileList:
supported = check_filename(file, supported_file_types)
if supported is True:
isStamped = openAndCheckFile(file)
if isStamped is False:
if check_filename(file, supported_file_types):
if needStampCheck(file):
unstampedFiles.append(file)
elif check_filename(file, unsupported_file_types):
unsupportedFiles.append(file)
else:
unsupported = check_filename(file, unsupported_file_types)
if unsupported is True:
unsupportedFiles.append(file)
else:
unknownFiles.append(file)
unknownFiles.append(file)
#Do a bunch of checks based on our file lists
# Do a bunch of checks based on our file lists
if len(unstampedFiles) > 0:
print("Error: The following " + str(len(unstampedFiles)) +
print("\nError: The following " + str(len(unstampedFiles)) +
" files are currently without a license:")
print(str(unstampedFiles))
sys.exit(1)
if len(unknownFiles) > 0:
print("Error: The following " + str(len(unknownFiles)) +
print("\nError: The following " + str(len(unknownFiles)) +
" files not handled:")
print(str(unknownFiles))
sys.exit(2)
sys.exit(0)
if __name__ == "__main__":
main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment