Skip to content

GitLab

Commit ac04f3cc authored by Khalique Ahmed's avatar Khalique Ahmed
Browse files

manual_merge

parents d39c3343 d8011adf
Hide whitespace changes
Inline Side-by-side
Showing with 395 additions and 97 deletions
src/include/migraphx/op/allocate.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -33,23 +33,83 @@ namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
/**
* Static allocate:
* No inputs: `allocate()`
* `this.s` attribute set to the static output shape of the buffer.
* `this.s` attribute can be set to a dynamic output shape; however this will allocate the maximum
* buffer size for that case
*
* Dynamic allocate:
* One input: `allocate(output_dims)`
* `output_dims` are the output buffer dimensions and has a static shape.
* Either `this.s` or `this.buf_type` (but not both) must be set to calculate the dynamic output
* shape at compute time. If `this.buf_type` is set, the compute_shape() of allocate at compile time
* will have dynamic_dimensions from {0, max_int} with rank = output_dims.ndim(). If `this.s` is set
* then the compute_shape() will output `this.s`; `this.s` should be a dynamic shape.
*/
struct allocate
{
shape s{};
optional<shape> s;
// for dynamic allocate to set the buffer type
optional<shape::type_t> buf_type;
template <class Self, class F>
static auto reflect(Self& self, F f)
{
return pack(f(self.s, "shape"));
return pack(f(self.s, "shape"), f(self.buf_type, "buf_type"));
}
std::string name() const { return "allocate"; }
shape compute_shape(const std::vector<shape>& inputs) const
{
migraphx::check_shapes{inputs, *this, true}.has(0);
return s;
if(s.has_value())
{
if(buf_type.has_value())
{
MIGRAPHX_THROW("ALLOCATE: shape and buf_type attributes both set");
}
if(inputs.size() == 1)
{
migraphx::check_shapes{inputs, *this, false}.only_dims(1);
}
else
{
migraphx::check_shapes{inputs, *this, false}.has(0);
}
return s.value();
}
else
{
if(not buf_type.has_value())
{
MIGRAPHX_THROW("ALLOCATE: shape and buf_type attributes both not set");
}
migraphx::check_shapes{inputs, *this, false}.has(1).only_dims(1);
const auto& out_dims = inputs.at(0);
std::size_t max_val = std::numeric_limits<std::size_t>::max();
std::vector<shape::dynamic_dimension> dyn_dims(out_dims.lens().at(0),
shape::dynamic_dimension{0, max_val});
return {buf_type.value(), dyn_dims};
}
}
argument compute(const shape& output_shape, const std::vector<argument>&) const
argument compute(const shape& output_shape, const std::vector<argument>& args) const
{
return {output_shape};
if(args.empty())
{
return argument{output_shape};
}
else
{
std::vector<std::size_t> output_dims(output_shape.ndim());
args.at(0).visit([&](auto a) { output_dims.assign(a.begin(), a.end()); });
if(s)
{
return argument{shape{s->type(), output_dims}};
}
return argument{shape{buf_type.value(), output_dims}};
}
}
};
......
src/include/migraphx/op/argmax.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -31,6 +31,7 @@
#include <migraphx/value.hpp>
#include <migraphx/op/normalize_attribute.hpp>
#include <migraphx/dyn_output.hpp>
#include <migraphx/float_equal.hpp>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
......@@ -38,12 +39,13 @@ namespace op {
struct argmax
{
int64_t axis = 0;
int64_t axis = 0;
bool select_last_index = false;
template <class Self, class F>
static auto reflect(Self& self, F f)
{
return pack(f(self.axis, "axis"));
return pack(f(self.axis, "axis"), f(self.select_last_index, "select_last_index"));
}
value attributes() const
......@@ -87,6 +89,10 @@ struct argmax
max_val = cur_val;
max_index = i;
}
else if(select_last_index and float_equal(max_val, cur_val))
{
max_index = i;
}
}
return max_index;
}
......
src/include/migraphx/op/argmin.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -30,6 +30,7 @@
#include <migraphx/config.hpp>
#include <migraphx/value.hpp>
#include <migraphx/op/normalize_attribute.hpp>
#include <migraphx/float_equal.hpp>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
......@@ -38,11 +39,12 @@ namespace op {
struct argmin
{
int64_t axis = 0;
bool select_last_index = false;
template <class Self, class F>
static auto reflect(Self& self, F f)
{
return pack(f(self.axis, "axis"));
return pack(f(self.axis, "axis"), f(self.select_last_index, "select_last_index"));
}
value attributes() const
......@@ -78,6 +80,10 @@ struct argmin
min_val = cur_val;
min_index = i;
}
else if(select_last_index and float_equal(min_val, cur_val))
{
min_index = i;
}
}
return min_index;
......
src/include/migraphx/op/common.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -33,8 +33,12 @@ namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
// Specifies where to add the "extra" cell of padding if the
// calculated padding is an odd number.
// Padding mode is default_ for fixed shape padding.
// same_lower and same_upper used for dynamic padding.
// same_lower and same_upper specify dynamic padding.
// The odd cell goes at the beginning of the dimension
// (same_lower) or end (same_upper).
enum padding_mode_t
{
default_, // NOLINT
......
src/include/migraphx/op/contiguous.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......
src/include/migraphx/op/convert.hpp
View file @ ac04f3cc
......@@ -68,7 +68,7 @@ struct convert : unary<convert>
auto y = x;
shape::visit(type, [&](auto as) {
// clamping value between target_type's max and min doesn't work for NaNs,
if(std::isnan(x))
if(std::isnan(static_cast<double>(x)))
{
y = as.nan();
}
......
src/include/migraphx/op/convolution.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -206,6 +206,7 @@ struct convolution
std::vector<std::size_t> new_padding;
if(padding_mode != op::padding_mode_t::default_)
{
// auto-Calculate the padding sizes with calc_dyn_auto_pad
auto input_lens = args[0].get_shape().lens();
auto weights_lens = args[1].get_shape().lens();
new_padding =
......@@ -217,6 +218,7 @@ struct convolution
}
else
{
// Use the padding that was given
new_padding = padding;
if(output_shape.dynamic())
{
......
src/include/migraphx/op/convolution_backwards.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -164,7 +164,7 @@ struct convolution_backwards
shape win_shape{dyn_out.computed_shape.type(), win_size};
par_dfor(in_n, wei_c)([&](int o, int k) {
shape_for_each(win_shape, [&](auto idx_win) {
shape_for_each(win_shape, [&](const auto& idx_win) {
const int w = idx_win[0];
auto input_dims_start = idx_win.begin() + 1;
......
src/include/migraphx/op/fill.hpp 0 → 100644
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MIGRAPHX_GUARD_OPERATORS_FILL_HPP
#define MIGRAPHX_GUARD_OPERATORS_FILL_HPP
#include <migraphx/check_shapes.hpp>
#include <migraphx/dyn_output.hpp>
#include <migraphx/par_for.hpp>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
/**
* fill(default_value, output_buffer)
* Fill an output buffer with the given default_value.
* Note that if the default_value is a literal and the output_buffer
* has a static shape this operator can be replaced with a literal.
*/
struct fill
{
std::string name() const { return "fill"; }
shape compute_shape(std::vector<shape> inputs) const
{
check_shapes{inputs, *this, true}.has(2).same_type();
if(inputs.at(0).dynamic() or inputs.at(0).elements() != 1)
{
MIGRAPHX_THROW("FILL: default_value is dynamic or more than one element");
}
return inputs.back();
}
argument compute(const dyn_output& dyn_out, std::vector<argument> args) const
{
visit_all(args[0], args[1])([&](auto value, auto output) {
par_for(dyn_out.computed_shape.elements(), [&](auto i) { output[i] = value.front(); });
});
return args[1];
}
std::ptrdiff_t output_alias(const std::vector<shape>&) const { return 1; }
};
} // namespace op
} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx
#endif
src/include/migraphx/op/gather.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -125,13 +125,12 @@ struct gather
auto out_lens = data.get_shape().lens();
out_lens[axis] = indices.get_shape().elements();
migraphx::shape out_comp_shape{data.get_shape().type(), out_lens};
shape_for_each(out_comp_shape, [&](const auto& out_idx) {
auto data_idx = out_idx;
auto in_index = indices[data_idx[axis]];
in_index = (in_index < 0) ? in_index + axis_dim_size : in_index;
data_idx[axis] = in_index;
output[out_comp_shape.index(out_idx.begin(), out_idx.end())] =
data(data_idx.begin(), data_idx.end());
shape_for_each(out_comp_shape, [&](const auto& out_idx_v, size_t out_idx) {
auto data_idx = out_idx_v;
auto in_index = indices[data_idx[axis]];
in_index = (in_index < 0) ? in_index + axis_dim_size : in_index;
data_idx[axis] = in_index;
output[out_idx] = data(data_idx.begin(), data_idx.end());
});
}
});
......
src/targets/gpu/include/migraphx/gpu/int8_conv_pack.hpp → src/include/migraphx/op/isinf.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -21,31 +21,32 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MIGRAPHX_GUARD_RTGLIB_INT8_CONV_PACK_HPP
#define MIGRAPHX_GUARD_RTGLIB_INT8_CONV_PACK_HPP
#ifndef MIGRAPHX_GUARD_OPERATORS_ISINF_HPP
#define MIGRAPHX_GUARD_OPERATORS_ISINF_HPP
#include <migraphx/argument.hpp>
#include <migraphx/op/unary.hpp>
#include <migraphx/config.hpp>
#include <utility>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace gpu {
namespace op {
struct context;
struct miopen_int8_conv_pack
struct isinf : unary<isinf>
{
std::string name() const { return "gpu::int8_conv_pack"; }
shape compute_shape(const std::vector<shape>& inputs) const;
argument compute(context& ctx, const shape&, const std::vector<argument>& args) const;
std::ptrdiff_t output_alias(const std::vector<shape>& shapes) const
auto apply() const
{
return [&](auto x) { return std::isinf(static_cast<double>(x)); };
}
std::string name() const { return "isinf"; }
shape compute_shape(std::vector<shape> inputs) const
{
return shapes.size() - 1;
return unary<isinf>::compute_shape(std::move(inputs)).with_type(shape::bool_type);
}
};
} // namespace gpu
} // namespace op
} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx
......
src/include/migraphx/op/isnan.hpp
View file @ ac04f3cc
......@@ -35,7 +35,7 @@ struct isnan : unary<isnan>
{
auto apply() const
{
return [](auto x) { return std::isnan(x); };
return [](auto x) { return std::isnan(static_cast<double>(x)); };
}
std::string name() const { return "isnan"; }
......
src/include/migraphx/op/multinomial.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -21,11 +21,52 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
/**
* * Multinomial or categorical distribution. Performs a sampling of random input
* and returns a count of
* each category, or bucket. This does not require the standard multinomial
* distribution but instead takes a probability distribution, i.e. cumulative
* distribution function (CDF) as its first input.
*
* Inputs: args[0] - a tensor of probabilities for each category. Values are
* cumulative density function
* totals as provided by operation prefix_scan_sum. Values are
* cumulative probabilities (i.e. start with any set of numbers > 0
* and then apply prefix_scan_sum). Values do not need to be
* normalized to sum to 1; this is done in runtime computation.
*
* This input has Rank 2. Dimension 0 is batch #, so that there can be
* a different CDF for each iteration in the batch. The size of dimension
* 1 is the number of categories.
*
* args[1] - a tensor of random numbers. The last dimension is the sample
* size, i.e. the number of
* random samples in each iteration of the batch. Nominally
* has two dimensions where the first dimension is batch size, but
* any reshaping such that the total
* number of elements is (batch_size * sample_size) is legal.
*
* Values as created by a std::mt19937 like this:
*
* size_t sample_size = 100000;
* float seed = 0.0f;
* std::mt19937 gen(seed);
* std::uniform_real_distribution<> dis(0.0, 1.0);
* std::vector<float> rand_samples(sample_size);
* std::generate(rand_samples.begin(), rand_samples.end(), [&]() { return
* dis(gen); });
*
* Output: A 2D vector of category each input. Dimensions are (Input 1[first], Input
2[last]).
*
*/
#ifndef MIGRAPHX_GUARD_OPERATORS_MULTINOMIAL_HPP
#define MIGRAPHX_GUARD_OPERATORS_MULTINOMIAL_HPP
#include <migraphx/check_shapes.hpp>
#include <migraphx/argument.hpp>
#include <migraphx/check_shapes.hpp>
#include <migraphx/dyn_output.hpp>
#include <migraphx/par_for.hpp>
#include <migraphx/reflect.hpp>
#include <random>
......@@ -47,22 +88,35 @@ struct multinomial
std::string name() const { return "multinomial"; }
shape compute_shape(std::vector<shape> inputs) const
{
check_shapes{inputs, *this}.has(2).only_dims(2);
size_t sample_size = inputs.back().lens().back();
check_shapes{inputs, *this, true}.has(2).only_dims(2);
if(not contains({shape::int32_type, shape::int64_type}, dtype))
MIGRAPHX_THROW(
"Multinomial: Invalid output type. Valid types are int32_type and int64_type.");
if(inputs.back().ndim() < 1)
MIGRAPHX_THROW("Multinomial: Second input shape (sample) has no dimensions");
if(dtype == shape::bool_type)
MIGRAPHX_THROW("Multinomial: boolean output type invalid.");
return {dtype, {inputs.front().lens().front(), sample_size}};
// Output takes one dimension from each of the two input shapes. If they are both fixed,
// return a static shape
if((not inputs.front().dynamic()) or (inputs.front().dyn_dims().front().is_fixed()))
{
if((not inputs.back().dynamic()) or (inputs.back().dyn_dims().back().is_fixed()))
{
size_t batch = {inputs.front().max_lens().front()};
size_t sample_size{inputs.back().max_lens().back()};
return {dtype, {batch, sample_size}};
}
}
return {dtype,
{inputs.front().to_dynamic().dyn_dims().front(),
inputs.back().to_dynamic().dyn_dims().back()}};
}
argument compute(const shape& output_shape, std::vector<argument> args) const
argument compute(const dyn_output& dyn_out, std::vector<argument> args) const
{
argument result{output_shape};
size_t batch_size = output_shape.lens().front();
argument result{dyn_out.computed_shape};
size_t batch_size = dyn_out.computed_shape.lens().front();
size_t class_size = args[0].get_shape().lens().back();
size_t sample_size = output_shape.lens().back();
size_t sample_size = dyn_out.computed_shape.lens().back();
visit_all(args[0], args[1])([&](auto cdf, auto dist) {
result.visit([&](auto output) {
......@@ -70,13 +124,16 @@ struct multinomial
auto idx = args[1].get_shape().multi(i);
auto cdf_begin = cdf.begin() + (idx[0] * class_size);
auto cdf_end = cdf_begin + class_size;
// std::upper_bound returns an iterator to the bucket the value belongs in,
// when normalized by the probability distribution dist
auto sample_iter =
std::upper_bound(cdf_begin, cdf_end, dist[i] * *(std::prev(cdf_end)));
// convert iterator to an integer index
output[i] = std::distance(cdf_begin, sample_iter);
});
});
});
return result;
}
};
......
src/include/migraphx/op/round.hpp → src/include/migraphx/op/nearbyint.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -21,24 +21,28 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
#ifndef MIGRAPHX_GUARD_OPERATORS_ROUND_HPP
#define MIGRAPHX_GUARD_OPERATORS_ROUND_HPP
#ifndef MIGRAPHX_GUARD_OPERATORS_NEARBYINT_HPP
#define MIGRAPHX_GUARD_OPERATORS_NEARBYINT_HPP
#include <migraphx/op/unary.hpp>
#include <migraphx/config.hpp>
#include <fenv.h>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
struct round : unary<round>
struct nearbyint : unary<nearbyint>
{
auto apply() const
{
return [](auto x) { return std::round(x); };
return [](auto x) {
auto rounding_mode = fegetround();
fesetround(FE_TONEAREST);
return std::nearbyint(x);
fesetround(rounding_mode);
};
}
};
} // namespace op
} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx
......
src/include/migraphx/op/nonmaxsuppression.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -24,6 +24,7 @@
#ifndef MIGRAPHX_GUARD_OPERATORS_NONMAXSUPPRESSION_HPP
#define MIGRAPHX_GUARD_OPERATORS_NONMAXSUPPRESSION_HPP
#include <array>
#include <cmath>
#include <queue>
#include <cstdint>
......@@ -258,7 +259,7 @@ struct nonmaxsuppression
selected_boxes_inside_class.reserve(max_output_shape.elements());
// iterate over batches and classes
shape comp_s{shape::double_type, {num_batches, num_classes}};
shape_for_each(comp_s, [&](auto idx) {
shape_for_each(comp_s, [&](const auto& idx) {
auto batch_idx = idx[0];
auto class_idx = idx[1];
// index offset for this class
......
src/include/migraphx/op/nonzero.hpp
View file @ ac04f3cc
/*
* The MIT License (MIT)
*
* Copyright (c) 2015-2022 Advanced Micro Devices, Inc. All rights reserved.
* Copyright (c) 2015-2023 Advanced Micro Devices, Inc. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
......@@ -56,10 +56,10 @@ struct nonzero
std::vector<std::vector<std::size_t>> vec_idx;
auto s = args.front().get_shape();
args.front().visit([&](auto v) {
shape_for_each(s, [&](auto idx) {
if(not float_equal(v[s.index(idx)], 0))
shape_for_each(s, [&](const auto& idx_v, size_t idx) {
if(not float_equal(v[idx], 0))
{
vec_idx.push_back(idx);
vec_idx.push_back(idx_v);
}
});
});
......
src/include/migraphx/op/normalize_attribute.hpp
View file @ ac04f3cc
......@@ -40,6 +40,8 @@ namespace op {
* 2. use_rank (default) vs use_len:
* `use_rank` sets the max value/index of the attribute as the rank of lens.
* `use_lens` sets the max value/index as the corresponding value in lens at the axes index.
* Uses the dynamic_dimension.max value for dynamic shapes. Returns the original vector
* (no normalization) if any of dynamic_dimension[axes] are not fixed.
* 3. `clip_min` vs. `not_clip_min` (default):
* Clip values less than the minimum to the minimum or not.
* 4. `include_min` vs. `exclude_min` (default):
......
src/include/migraphx/op/pooling.hpp
View file @ ac04f3cc
......@@ -29,6 +29,7 @@
#include <migraphx/config.hpp>
#include <migraphx/value.hpp>
#include <migraphx/argument.hpp>
#include <migraphx/pad_calc.hpp>
#include <migraphx/par_for.hpp>
#include <migraphx/shape_for_each.hpp>
#include <migraphx/dyn_output.hpp>
......@@ -40,10 +41,20 @@ namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
// The Pooling operator mostly follows the specifications for the Onnx pooling op.
// It assumes an NCHW layout, extended to support any number of spatial dimensions
// from 1 on up; dimensions are <batch index, channels, spatial dimensions...>
//
struct pooling
{
// Class members mode, ceil_mode, padding_mode have similar names but refer to separate
// concepts.
pooling_mode mode = {pooling_mode::average};
// If the input has rank other than 4 then padding, stride, lengths must all be specified
// since the defaults have 2-dimensions. Exception: padding not required if
// padding_mode != default_
// Padding along each spatial input dimension
// Can be ndim or 2*ndim values where ndim is size of lengths
// ndim values means pad the same before and after each dimension
......@@ -63,13 +74,14 @@ struct pooling
// ceiling mode is a flag affecting output size
// or equivalently, placements of the pooling kernel.
// When true, round the size upwards, possibly
// including partial placements where the kernel extends beyond the edge
// of input and even padding. When false, round down so that all
// When true, round the size upwards. When false, round down so that all
// kernel placements fit but some input values may be dropped.
bool ceil_mode = false;
int lp_order = 2;
// Mode for auto padding. default_ indicates no auto padding.
padding_mode_t padding_mode = padding_mode_t::default_;
// Global pooling with dynamic shape input
bool dyn_global = false;
......@@ -84,6 +96,7 @@ struct pooling
{
return pack(f(self.mode, "mode"),
f(self.padding, "padding"),
f(self.padding_mode, "padding_mode"),
f(self.stride, "stride"),
f(self.lengths, "lengths"),
f(self.ceil_mode, "ceil_mode"),
......@@ -97,7 +110,8 @@ struct pooling
{
if(dyn_global)
return;
if((padding.size() != stride.size() and (padding.size()) != stride.size() * 2) or
if((padding_mode != default_ and padding.size() != stride.size() and
(padding.size()) != stride.size() * 2) or
stride.size() != lengths.size())
{
MIGRAPHX_THROW("POOLING: inconsistent attribute sizes");
......@@ -137,8 +151,19 @@ struct pooling
std::size_t padding_factor = 2 * padding[i];
if(padding.size() == 2 * kdims)
padding_factor = padding[i] + padding[i + kdims];
assert(input_lens[i + 2] + padding_factor >= lengths[i]);
std::size_t dim_size = input_lens[i + 2] + padding_factor - lengths[i];
std::size_t dim_size;
if(input_lens[i + 2] + padding_factor < lengths[i])
{
if(padding_mode == default_)
MIGRAPHX_THROW("POOLING: not enough padding for the given kernel size");
// lengths can be legitimately larger only if we're doing auto padding
// with a dynamic shape, in which case given padding is ignored. Set a dummy value.
dim_size = 2;
}
else
{
dim_size = input_lens[i + 2] + padding_factor - lengths[i];
}
std::size_t len =
(ceil_mode)
? dim_size / stride[i] +
......@@ -151,17 +176,13 @@ struct pooling
shape normalize_compute_shape(std::vector<shape> inputs) const
{
check_shapes{inputs, *this, true}.has(1);
check_shapes{inputs, *this, true}.has(1).min_ndims(3);
check_attribute_size();
const shape& input = inputs.at(0);
auto padding_size = padding.size();
auto stride_size = stride.size();
size_t kdims = input.ndim() - 2;
if(input.ndim() < 3)
{
MIGRAPHX_THROW("POOLING: input must have 3 or more dimensions and be nonempty");
}
if(input.ndim() * 2 != padding_size + 4 and input.ndim() != padding_size + 2)
if(input.ndim() != stride_size + 2)
{
MIGRAPHX_THROW("POOLING: input and attribute size mismatch!");
}
......@@ -179,6 +200,28 @@ struct pooling
}
return {input.type(), output_dyn_dims};
}
else if(padding_mode != default_)
{
const size_t num_spatial_dims = inputs[0].ndim() - 2;
const shape& x_shape = inputs[0];
// same as convolution::dynamic_compute_shape()
for(std::size_t i = 0; i < num_spatial_dims; ++i)
{
auto ceil_div = [](std::size_t x, std::size_t y) { return (x + y - 1) / y; };
auto s = stride[i];
auto x = x_shape.dyn_dims()[i + 2];
std::set<std::size_t> optimals{};
std::transform(x.optimals.begin(),
x.optimals.end(),
std::inserter(optimals, optimals.begin()),
[&](auto o) { return ceil_div(o, s); });
output_dyn_dims.push_back(
shape::dynamic_dimension{ceil_div(x.min, s), ceil_div(x.max, s), optimals});
}
return {input.type(), output_dyn_dims};
}
else
{
// does not compute optimals
......@@ -267,6 +310,7 @@ struct pooling
Out& output,
const In& input,
const std::vector<std::size_t>& kernel_dims,
const std::vector<std::size_t>& padding_vals,
Op op) const
{
auto in_s = input.get_shape();
......@@ -283,9 +327,9 @@ struct pooling
// For each spatial dimension, find starting and ending index of pooling kernel
for(std::size_t dim = 2; dim < n_dim; ++dim)
{
auto d_2 = dim - 2;
int start =
static_cast<int>(idx_o[dim] * stride[d_2]) - static_cast<int>(padding[d_2]);
auto d_2 = dim - 2;
int start = static_cast<int>(idx_o[dim] * stride[d_2]) -
static_cast<int>(padding_vals[d_2]);
int end;
// NOLINT
if(count_include_pad and ceil_mode and (mode != pooling_mode::max))
......@@ -297,7 +341,7 @@ struct pooling
// Check if this kernel extends beyond the padding at end of dimension
end = std::min(start + kernel_dims[d_2],
in_lens[dim] + static_cast<int>(padding[d_2]));
in_lens[dim] + static_cast<int>(padding_vals[d_2]));
}
else
{
......@@ -316,11 +360,12 @@ struct pooling
}
shape win_shape{output_shape.type(), win_size};
auto pool_size = win_shape.elements();
double output_val = op.template init<Type>();
// for each element in the window...
shape_for_each(win_shape, [&](auto idx_w) {
shape_for_each(win_shape, [&](const auto& idx_w) {
// the coordinates of this element
auto idx = idx_o;
......@@ -354,30 +399,65 @@ struct pooling
argument compute(const dyn_output& dyn_out, std::vector<argument> args) const
{
argument result{dyn_out.computed_shape};
argument result;
auto input_lens = args[0].get_shape().lens();
std::vector<std::size_t> kernel_dims;
shape output_shape;
// If we have to auto-calculate padding, it will be passed to calc_pooling() as an argument
// instead of the member variable padding.
std::vector<std::size_t> temp_padding(padding);
if(dyn_global)
{
// for dynamic GlobalPooling, there's no padding
kernel_dims.insert(kernel_dims.end(), input_lens.begin() + 2, input_lens.end());
output_shape = dyn_out.computed_shape;
result = argument{dyn_out.computed_shape};
}
else
else if((padding_mode != op::padding_mode_t::default_))
{
// if padding_mode is set, input was a dynamic size. Calculate padded size now.
// kernel_lens is the same as kernel_dims, but prepended with the 2 non-
// spatial dimensions. For size computations, it's used like the weights
// tensor for convolutions.
std::vector<std::size_t> kernel_lens;
kernel_lens.insert(kernel_lens.end(), input_lens.begin(), input_lens.begin() + 2);
kernel_lens.insert(kernel_lens.end(), lengths.begin(), lengths.end());
kernel_dims = this->lengths;
auto type = args[0].get_shape().type();
// dilation not currently supported for pooling, so default to all 1's
temp_padding = calc_dyn_auto_pad(
input_lens, kernel_lens, stride, {1, 1}, bool(padding_mode == op::same_upper));
output_shape = compute_padded_pool_shape(
args[0].get_shape(), shape(type, kernel_dims), temp_padding, stride, {1, 1});
result = argument(output_shape);
}
else // fixed/static input
{
kernel_dims = this->lengths;
output_shape = dyn_out.computed_shape;
result = argument{dyn_out.computed_shape};
}
// Perform the computation and populate result
visit_all(result, args[0])([&](auto output, auto input) {
using type = typename decltype(output)::value_type;
switch(mode)
{
case migraphx::op::pooling_mode::average:
calc_pooling<type>(dyn_out.computed_shape, output, input, kernel_dims, avg_pool{});
calc_pooling<type>(
output_shape, output, input, kernel_dims, temp_padding, avg_pool{});
break;
case migraphx::op::pooling_mode::max:
calc_pooling<type>(dyn_out.computed_shape, output, input, kernel_dims, max_pool{});
calc_pooling<type>(
output_shape, output, input, kernel_dims, temp_padding, max_pool{});
break;
case migraphx::op::pooling_mode::lpnorm:
calc_pooling<type>(
dyn_out.computed_shape, output, input, kernel_dims, lpnorm_pool{lp_order});
output_shape, output, input, kernel_dims, temp_padding, lpnorm_pool{lp_order});
break;
}
});
......
src/include/migraphx/op/prefix_scan_op.hpp
View file @ ac04f3cc
......@@ -22,6 +22,12 @@
* THE SOFTWARE.
*/
/**
* Parent struct for prefix scan ops. A prefix scan is a mathematical entity useful
* in parallelizing various computations. Given a list of numbers, a prefix scan
* op returns an equal size list of running totals of the values. Other operations
* besides addition can be supported by child ops.
*/
#ifndef MIGRAPHX_GUARD_OPERATORS_SCAN_OP_HPP
#define MIGRAPHX_GUARD_OPERATORS_SCAN_OP_HPP
......
src/include/migraphx/op/quantizelinear.hpp
View file @ ac04f3cc
......@@ -30,11 +30,11 @@
#include <migraphx/par_for.hpp>
#include <migraphx/value.hpp>
#include <cmath>
#include <fenv.h>
namespace migraphx {
inline namespace MIGRAPHX_INLINE_NS {
namespace op {
struct quantizelinear
{
std::string name() const { return "quantizelinear"; }
......@@ -71,26 +71,26 @@ struct quantizelinear
{
y_zero_point = args.at(2);
}
argument result{output_shape};
auto rounding_mode = fegetround();
fesetround(FE_TONEAREST);
visit_all(result, y_zero_point)([&](auto output, auto zero_pts) {
visit_all(x, y_scale)([&](auto input, auto scales) {
using quant_type = typename decltype(output)::value_type;
auto min_value = std::numeric_limits<quant_type>::min();
auto max_value = std::numeric_limits<quant_type>::max();
par_for(output_shape.elements(), [&](auto i) {
int64_t quantized = static_cast<int64_t>(std::round(input[i] / scales[i])) +
int64_t quantized = static_cast<int64_t>(std::nearbyint(input[i] / scales[i])) +
static_cast<int64_t>(zero_pts[i]);
output[i] = std::max(static_cast<int64_t>(min_value),
std::min(static_cast<int64_t>(max_value), quantized));
});
});
});
fesetround(rounding_mode);
return result;
}
};
} // namespace op
} // namespace MIGRAPHX_INLINE_NS
} // namespace migraphx
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment