Merge remote-tracking branch 'upstream/develop' into gemm-layernorm-4

include/ck/utility/reduction_functions_accumulate.hpp

-/*******************************************************************************
- *
- * MIT License
- *
- * Copyright (c) 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- *******************************************************************************/
-#ifndef CK_REDUCTION_FUNCTIONS_BINOP_HPP
-#define CK_REDUCTION_FUNCTIONS_BINOP_HPP
-
-#include "data_type.hpp"
-#include "math_v2.hpp"
-
-#include "reduction_common.hpp"
-#include "reduction_operator.hpp"
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include "ck/utility/data_type.hpp"
+#include "ck/utility/math_v2.hpp"
+#include "ck/utility/reduction_common.hpp"
+#include "ck/utility/reduction_operator.hpp"
 
 namespace ck {
 namespace detail {
 
+// Check for NaN; guarantee NaNs are NOT propagated to result (i.e., ignore NaNs)
+template <typename ReduceOperation, typename AccDataType>
+struct AccumulateWithNanIgnore
+{
+    __device__ static inline void Calculate(AccDataType& accuVal, AccDataType currVal)
+    {
+        if(!isnan(currVal))
+        {
+            ReduceOperation{}(accuVal, currVal);
+        }
+    };
+};
+
 template <bool PropagateNan, typename ReduceOperation, typename AccDataType>
 struct AccumulateWithNanCheck;
 
+// Does not check for NaN; does not guarantee NaNs be propagated to result
+// e.g., given that max(a, b) = a > b ? a : b
+// then  max(NaN, 1) returns 1
+//       max(1, NaN) returns NaN
+// since any comparison involving NaNs returns false
 template <typename ReduceOperation, typename AccDataType>
 struct AccumulateWithNanCheck<false, ReduceOperation, AccDataType>
 {
@@ -48,6 +42,7 @@ struct AccumulateWithNanCheck<false, ReduceOperation, AccDataType>
     };
 };
 
+// Check for NaN; guarantees NaNs be propagated to result
 template <typename ReduceOperation, typename AccDataType>
 struct AccumulateWithNanCheck<true, ReduceOperation, AccDataType>
 {
@@ -116,7 +111,5 @@ struct AccumulateWithIndexAndNanCheck<true, ReduceOperation, AccDataType, IndexD
     };
 };
 
-}; // namespace detail
-}; // end of namespace ck
-
-#endif
+} // namespace detail
+} // namespace ck

include/ck/utility/reduction_operator.hpp

-/*******************************************************************************
- *
- * MIT License
- *
- * Copyright (c) 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- *******************************************************************************/
-#ifndef CK_REDUCTION_OPERATOR_HPP
-#define CK_REDUCTION_OPERATOR_HPP
-
-#include "config.hpp"
-#include "data_type.hpp"
-#include "type.hpp"
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include "ck/ck.hpp"
+#include "ck/utility/data_type.hpp"
+#include "ck/utility/type.hpp"
 
 namespace ck {
 
@@ -311,8 +288,5 @@ struct InMemoryDataOperatonSupportedOnDataType<InMemoryDataOperationEnum::Add, D
         is_same<DataType, int32_t>::value;
 };
 
-}; // end of namespace reduce
-
-} // end of namespace ck
-
-#endif
+} // namespace reduce
+} // namespace ck

include/ck/utility/sequence.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
 
 #include "integral_constant.hpp"

include/ck/utility/sequence_helper.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #ifndef CK_SEQUENCE_HELPER_HPP
 #define CK_SEQUENCE_HELPER_HPP
 

include/ck/utility/static_buffer.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #ifndef CK_STATIC_BUFFER_HPP
 #define CK_STATIC_BUFFER_HPP
 

include/ck/utility/statically_indexed_array.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #ifndef CK_STATICALLY_INDEXED_ARRAY_HPP
 #define CK_STATICALLY_INDEXED_ARRAY_HPP
 

include/ck/utility/statically_indexed_array_multi_index.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #ifndef CK_STATICALLY_INDEXED_ARRAY_MULTI_INDEX_HPP
 #define CK_STATICALLY_INDEXED_ARRAY_MULTI_INDEX_HPP
 

include/ck/utility/synchronization.hpp

-#ifndef CK_SYNCHRONIZATION_AMD_HPP
-#define CK_SYNCHRONIZATION_AMD_HPP
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "config.hpp"
+#pragma once
+
+#include "ck/ck.hpp"
 
 namespace ck {
 
@@ -18,4 +20,3 @@ __device__ void block_sync_lds()
 }
 
 } // namespace ck
-#endif

include/ck/utility/thread_group.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
+
 #include "get_id.hpp"
 
 namespace ck {

include/ck/utility/transpose_vectors.hpp

-#ifndef CK_TRANSPOSE_VECTORS_AMD_HPP
-#define CK_TRANSPOSE_VECTORS_AMD_HPP
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "config.hpp"
+#pragma once
+
+#include "ck/ck.hpp"
 #include "statically_indexed_array.hpp"
 #include "data_type.hpp"
 
@@ -165,4 +167,3 @@ struct transpose_vectors<int8_t, NX, NY>
 };
 
 } // namespace ck
-#endif

include/ck/utility/tuple.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
 
 #include "integral_constant.hpp"

include/ck/utility/tuple_helper.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
 
 #include "functional4.hpp"

include/ck/utility/type.hpp

-#ifndef CK_TYPE_HPP
-#define CK_TYPE_HPP
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "config.hpp"
+#pragma once
+
+#include "ck/ck.hpp"
 #include "integral_constant.hpp"
 #include "enable_if.hpp"
 
@@ -56,4 +58,3 @@ __host__ __device__ constexpr Y bit_cast(const X& x)
 }
 
 } // namespace ck
-#endif

library/include/ck/library/host/host_interface.hpp

-#pragma once
-
-#include <memory>
-#include <string>
-
-#include "stream_config.hpp"
-#include "config.hpp"
-#include "device_base.hpp"
-
-struct DeviceConvFwdPtr_t
-{
-    using BaseArgument = ck::tensor_operation::device::BaseArgument;
-    using BaseInvoker  = ck::tensor_operation::device::BaseInvoker;
-
-    struct DeviceConvFwdPtrImpl;
-    std::unique_ptr<DeviceConvFwdPtrImpl> pImpl;
-    DeviceConvFwdPtr_t();
-    ~DeviceConvFwdPtr_t();
-    DeviceConvFwdPtr_t(DeviceConvFwdPtr_t&&);
-    DeviceConvFwdPtr_t(DeviceConvFwdPtrImpl&);
-    DeviceConvFwdPtr_t& operator=(DeviceConvFwdPtr_t&) = delete;
-    DeviceConvFwdPtr_t& operator=(const DeviceConvFwdPtr_t&) = delete;
-    std::unique_ptr<BaseArgument>
-    MakeArgumentPointer(void* in_ptr,
-                        void* wei_ptr,
-                        void* out_ptr,
-                        size_t N,
-                        size_t K,
-                        size_t C,
-                        std::vector<ck::index_t> input_spatial_lengths,
-                        std::vector<ck::index_t> filter_spatial_lengths,
-                        std::vector<ck::index_t> output_spatial_lengths,
-                        std::vector<ck::index_t> conv_filter_strides,
-                        std::vector<ck::index_t> conv_filter_dilations,
-                        std::vector<ck::index_t> input_left_pads,
-                        std::vector<ck::index_t> input_right_pads)
-        const; // in,wei and out element ops are ignored for now since even if we change them, they
-               // cant be linked
-    std::unique_ptr<BaseInvoker>
-    MakeInvokerPointer() const; // requires including BaseInvoker headers
-    std::string GetTypeString();
-    bool IsSupportedArgument(const BaseArgument* arg_ptr);
-};
-
-void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f32_instances_t(
-    std::vector<DeviceConvFwdPtr_t>& instances);
-void add_device_conv2d_fwd_xdl_c_shuffle_nhwc_kyxc_nhwk_f16_instances_t(
-    std::vector<DeviceConvFwdPtr_t>& instances);
-void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_bf16_instances_t(
-    std::vector<DeviceConvFwdPtr_t>& instances);
-void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_f16_instances_t(
-    std::vector<DeviceConvFwdPtr_t>& instances);
-void add_device_conv2d_fwd_xdl_nhwc_kyxc_nhwk_int8_instances_t(
-    std::vector<DeviceConvFwdPtr_t>& instances);

library/include/ck/library/host_tensor/conv_common.hpp

-#ifndef CONV_COMMON_HPP
-#define CONV_COMMON_HPP
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
 
-#include "tensor_descriptor.hpp"
+#pragma once
+
+#include "ck/tensor_description/tensor_descriptor.hpp"
 
 template <typename... InDesc,
           typename... WeiDesc,
@@ -73,18 +75,3 @@ calculate_convolution_flops(const InDesc&, const WeiDesc& wei_desc, const OutDes
 
     return std::size_t(2) * N * K * Ho * Wo * C * Y * X;
 }
-
-template <typename T>
-inline auto activ(T v, const ck::ActivTypeEnum activ_type)
-{
-    const T alpha = 0.3;
-    switch(activ_type)
-    {
-    case ck::ActivTypeEnum::None: return v;
-    case ck::ActivTypeEnum::LeakyRelu: return (v >= 0 ? v : alpha * v);
-    case ck::ActivTypeEnum::Sigmoid: return (1 / (1 + exp(-v)));
-    default: throw std::runtime_error("unsupported activ type"); break;
-    }
-}
-
-#endif

library/include/ck/library/host_tensor/device_memory.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
+
+#include <hip/hip_runtime.h>
+
+template <typename T>
+__global__ void set_buffer_value(T* p, T x, uint64_t buffer_element_size)
+{
+    for(uint64_t i = threadIdx.x; i < buffer_element_size; i += blockDim.x)
+    {
+        p[i] = x;
+    }
+}
+
+struct DeviceMem
+{
+    DeviceMem() = delete;
+    DeviceMem(std::size_t mem_size);
+    void* GetDeviceBuffer();
+    std::size_t GetBufferSize();
+    void ToDevice(const void* p);
+    void FromDevice(void* p);
+    void SetZero();
+    template <typename T>
+    void SetValue(T x)
+    {
+        if(mMemSize % sizeof(T) != 0)
+        {
+            throw std::runtime_error("wrong! not entire DeviceMem will be set");
+        }
+
+        set_buffer_value<T><<<1, 1024>>>(static_cast<T*>(mpDeviceBuf), x, mMemSize / sizeof(T));
+    }
+    ~DeviceMem();
+
+    void* mpDeviceBuf;
+    std::size_t mMemSize;
+};

library/include/ck/library/host_tensor/device_tensor.hpp

-#pragma once
-#include "host_tensor.hpp"
-
-template <typename TensorDesc>
-void ostream_tensor_descriptor(TensorDesc, std::ostream& os = std::cout)
-{
-    ostream_HostTensorDescriptor(make_HostTensorDescriptor(TensorDesc{}), os);
-}

library/include/ck/library/host_tensor/host_common_util.hpp

-/*******************************************************************************
- *
- * MIT License
- *
- * Copyright (c) 2020 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- *******************************************************************************/
-#ifndef GUARD_HOST_COMMON_UTIL_HPP
-#define GUARD_HOST_COMMON_UTIL_HPP
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
+#pragma once
 
 #include <vector>
 #include <iostream>
 #include <fstream>
 #include <string>
 
-#include "config.hpp"
+#include "ck/ck.hpp"
 
 namespace ck {
 
@@ -95,8 +72,5 @@ static inline std::vector<T> getTypeValuesFromString(const char* cstr_values)
     return (values);
 }
 
-}; // namespace host_common
-
-}; // namespace ck
-
-#endif
+} // namespace host_common
+} // namespace ck

library/include/ck/library/host_tensor/host_conv.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
 #include "host_tensor.hpp"
 #include "conv_common.hpp"

library/include/ck/library/host_tensor/host_gemm.hpp

+// SPDX-License-Identifier: MIT
+// Copyright (c) 2018-2022, Advanced Micro Devices, Inc. All rights reserved.
+
 #pragma once
+
 #include "host_tensor.hpp"
 
 template <typename AType,