Unverified Commit 69d8d789 authored by Umang Yadav's avatar Umang Yadav Committed by GitHub
Browse files

Add options to set tolerances inside MIGraphX driver (#2213)

MIGraphX verification by default uses normalized RMS error as the basis for the verification.  This change adds some logic to allow migraphx to do "np.allclose" type of elementwise verification using atol and rtol.

Commit also includes changes to consistently pass "gold" or "expected" results as the second argument for "verify_range()" calls.  Default RMS tolerance inside driver is set to 0.001 which IMO is high for FP32 compared to what we had earlier. Need better defaults
parent e12032fb
......@@ -47,7 +47,7 @@ TEST_CASE(argmin_test_0)
std::vector<int64_t> result_vec;
result.visit([&](auto output) { result_vec.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(result_vec, res_gold));
EXPECT(migraphx::verify::verify_rms_range(result_vec, res_gold));
}
TEST_CASE(argmin_test_1)
......@@ -66,7 +66,7 @@ TEST_CASE(argmin_test_1)
std::vector<int64_t> result_vec;
result.visit([&](auto output) { result_vec.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(result_vec, res_gold));
EXPECT(migraphx::verify::verify_rms_range(result_vec, res_gold));
}
TEST_CASE(argmin_test_2)
......@@ -85,7 +85,7 @@ TEST_CASE(argmin_test_2)
std::vector<int64_t> result_vec;
result.visit([&](auto output) { result_vec.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(result_vec, res_gold));
EXPECT(migraphx::verify::verify_rms_range(result_vec, res_gold));
}
TEST_CASE(argmin_test_neg_1)
......@@ -104,7 +104,7 @@ TEST_CASE(argmin_test_neg_1)
std::vector<int64_t> result_vec;
result.visit([&](auto output) { result_vec.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(result_vec, res_gold));
EXPECT(migraphx::verify::verify_rms_range(result_vec, res_gold));
}
TEST_CASE(argmin_test_nonstd_shape)
......@@ -123,5 +123,5 @@ TEST_CASE(argmin_test_nonstd_shape)
result.visit([&](auto output) { result_vec.assign(output.begin(), output.end()); });
std::vector<int64_t> res_gold_vec;
res_gold.visit([&](auto output) { res_gold_vec.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(result_vec, res_gold_vec));
EXPECT(migraphx::verify::verify_rms_range(result_vec, res_gold_vec));
}
......@@ -45,7 +45,7 @@ TEST_CASE(asin_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return asinf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(asin_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(asin_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return asinf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(asinh_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return asinhf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(asinh_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(asinh_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return asinhf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(atan_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return atanf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(atan_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(atan_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return atanf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(atanh_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return atanhf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(atanh_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(atanh_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return atanhf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -114,6 +114,6 @@ TEST_CASE(isnan_broadcast_test)
auto result = p.eval({}).back();
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> correct = {0, 0, 0, 0, 1, 1};
EXPECT(migraphx::verify::verify_range(results_vector, correct));
std::vector<float> gold = {0, 0, 0, 0, 1, 1};
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(ceil_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return std::ceil(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(ceil_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(ceil_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return std::ceil(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -48,7 +48,7 @@ TEST_CASE(clip_test)
std::vector<float> results_vector(3);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0.0, 0.0, 6.0};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(clip_dyn_test)
......@@ -73,5 +73,5 @@ TEST_CASE(clip_dyn_test)
std::vector<float> results_vector(3);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0.0, 0.0, 6.0};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -50,11 +50,11 @@ TEST_CASE(concat_test_1)
std::vector<int> gold = {0, 1, 2, 3, 4, 10, 5, 6, 7, 8, 9, 20};
std::vector<int> results_vector(2 * 6);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_range(result.get_shape().lens(),
std::vector<std::size_t>({2, 6})));
EXPECT(migraphx::verify::verify_range(result.get_shape().strides(),
std::vector<std::size_t>({6, 1})));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().lens(),
std::vector<std::size_t>({2, 6})));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().strides(),
std::vector<std::size_t>({6, 1})));
}
TEST_CASE(concat_test_2)
......@@ -77,11 +77,11 @@ TEST_CASE(concat_test_2)
std::vector<int> gold = {0, 1, 2, 3, 4, 10, 5, 6, 7, 8, 9, 20};
std::vector<int> results_vector(2 * 6);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_range(result.get_shape().lens(),
std::vector<std::size_t>({2, 6})));
EXPECT(migraphx::verify::verify_range(result.get_shape().strides(),
std::vector<std::size_t>({6, 1})));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().lens(),
std::vector<std::size_t>({2, 6})));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().strides(),
std::vector<std::size_t>({6, 1})));
}
TEST_CASE(concat_test_3)
......@@ -104,11 +104,11 @@ TEST_CASE(concat_test_3)
std::vector<int> gold = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
std::vector<int> results_vector(6 * 2);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
EXPECT(migraphx::verify::verify_range(result.get_shape().strides(),
std::vector<std::size_t>({2, 1})));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().strides(),
std::vector<std::size_t>({2, 1})));
}
TEST_CASE(concat_test_4)
......@@ -131,11 +131,11 @@ TEST_CASE(concat_test_4)
std::vector<int> gold = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
std::vector<int> results_vector(6 * 2);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
EXPECT(migraphx::verify::verify_range(result.get_shape().strides(),
std::vector<std::size_t>({2, 1})));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().strides(),
std::vector<std::size_t>({2, 1})));
}
TEST_CASE(concat_dyn_test)
......@@ -169,7 +169,7 @@ TEST_CASE(concat_dyn_test)
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(result.get_shape().lens(),
std::vector<std::size_t>({6, 2})));
}
......@@ -50,7 +50,7 @@ TEST_CASE(contiguous_test)
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(contiguous_param_test)
......@@ -74,7 +74,7 @@ TEST_CASE(contiguous_param_test)
std::vector<float> results_vector(12);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(contiguous_dyn_test)
......@@ -100,5 +100,5 @@ TEST_CASE(contiguous_dyn_test)
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold = {0, 3, 6, 9, 1, 4, 7, 10, 2, 5, 8, 11};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
This diff is collapsed.
......@@ -52,7 +52,7 @@ TEST_CASE(convolution_backwards_1d)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_2d)
......@@ -75,7 +75,7 @@ TEST_CASE(convolution_backwards_2d)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_3d)
......@@ -113,7 +113,7 @@ TEST_CASE(convolution_backwards_3d)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_padding1)
......@@ -139,7 +139,7 @@ TEST_CASE(convolution_backwards_padding1)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_padding2)
......@@ -165,7 +165,7 @@ TEST_CASE(convolution_backwards_padding2)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_2stride)
......@@ -192,7 +192,7 @@ TEST_CASE(convolution_backwards_2stride)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_2dilation)
......@@ -219,7 +219,7 @@ TEST_CASE(convolution_backwards_2dilation)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_dyn_batch1)
......@@ -247,7 +247,7 @@ TEST_CASE(convolution_backwards_dyn_batch1)
27, 15, 9, 20, 33, 24, 13, 6, 13, 21, 15, 8};
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(convolution_backwards_dyn_batch2)
......@@ -288,5 +288,5 @@ TEST_CASE(convolution_backwards_dyn_batch2)
std::vector<float> results_vector;
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(cos_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return cosf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(cos_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(cos_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return cosf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(cosh_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return coshf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(cosh_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(cosh_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return coshf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -47,7 +47,7 @@ TEST_CASE(dimensions_of_test0)
std::vector<int64_t> results_vector(3);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<int64_t> gold = {2, 3, 4};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(dimensions_of_test1)
......@@ -67,5 +67,5 @@ TEST_CASE(dimensions_of_test1)
std::vector<int64_t> results_vector(2);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<int64_t> gold = {4, 4};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -46,7 +46,7 @@ TEST_CASE(div_test)
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold(data1.size());
std::transform(data1.begin(), data1.end(), data2.begin(), gold.begin(), std::divides<float>());
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(div_dyn_test)
......@@ -72,5 +72,5 @@ TEST_CASE(div_dyn_test)
std::vector<float> gold(x_data.size());
std::transform(
x_data.begin(), x_data.end(), y_data.begin(), gold.begin(), std::divides<float>());
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
This diff is collapsed.
......@@ -45,7 +45,7 @@ TEST_CASE(elu_test)
std::vector<float> results_vector(4);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold{elu(alpha, -1), elu(alpha, 2), elu(alpha, -3), elu(alpha, 4)};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(elu_dyn_test)
......@@ -67,5 +67,5 @@ TEST_CASE(elu_dyn_test)
std::vector<float> results_vector(4);
result.visit([&](auto output) { results_vector.assign(output.begin(), output.end()); });
std::vector<float> gold{elu(alpha, -1), elu(alpha, 2), elu(alpha, -3), elu(alpha, 4)};
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(erf_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return erff(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(erf_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(erf_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return erff(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
......@@ -45,7 +45,7 @@ TEST_CASE(exp_test)
std::vector<float> gold = data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return expf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
TEST_CASE(exp_dyn_test)
......@@ -68,5 +68,5 @@ TEST_CASE(exp_dyn_test)
std::vector<float> gold = input_data;
std::transform(
gold.begin(), gold.end(), gold.begin(), [](float n) -> float { return expf(n); });
EXPECT(migraphx::verify::verify_range(results_vector, gold));
EXPECT(migraphx::verify::verify_rms_range(results_vector, gold));
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment