Skip to content

GitLab

"model.properties" did not exist on "6370e1258b71da29fd25f3e313cfae122893985c"
Unverified Commit 0a7174ad authored by Chao Liu's avatar Chao Liu Committed by GitHub
Browse files

Merge with (not the latest) upstream CK (#32) 

* fix build for old ck examples

* fix build for old ck
parent 496be40e
Show whitespace changes
Inline Side-by-side
Showing with 81 additions and 118 deletions
include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r1r2.hpp
View file @ 0a7174ad
......@@ -94,7 +94,7 @@ struct ThreadwiseTensorSliceTransfer_v6r1r2
using dst_vector_t = typename dst_vector_type::type;
const bool is_src_valid =
coordinate_has_valid_offset_assuming_visible_index_is_valid(src_desc, src_coord_);
coordinate_has_valid_offset_assuming_top_index_is_valid(src_desc, src_coord_);
// copy data from src_buf into src_vector_container
auto src_vector_container = src_vector_type{
......@@ -114,7 +114,7 @@ struct ThreadwiseTensorSliceTransfer_v6r1r2
});
const bool is_dst_valid =
coordinate_has_valid_offset_assuming_visible_index_is_valid(dst_desc, dst_coord_);
coordinate_has_valid_offset_assuming_top_index_is_valid(dst_desc, dst_coord_);
// copy data from dst_vector into dst_buf
dst_buf.template Update<DstInMemOp, dst_vector_t>(
......@@ -126,28 +126,20 @@ struct ThreadwiseTensorSliceTransfer_v6r1r2
if constexpr(idx_1d.value != num_access - 1)
{
constexpr auto forward_step = SpaceFillingCurve::GetForwardStep(idx_1d);
move_tensor_coordinate(
src_desc, src_coord_, make_tensor_coordinate_step(src_desc, forward_step));
move_tensor_coordinate(
dst_desc, dst_coord_, make_tensor_coordinate_step(dst_desc, forward_step));
move_tensor_coordinate(src_desc, src_coord_, forward_step);
move_tensor_coordinate(dst_desc, dst_coord_, forward_step);
}
});
// move coordinate back to slice origin (or not)
if constexpr(SrcResetCoordinateAfterRun)
{
const auto src_reset_step =
make_tensor_coordinate_step(src_desc, GetCoordinateResetStep());
move_tensor_coordinate(src_desc, src_coord_, src_reset_step);
move_tensor_coordinate(src_desc, src_coord_, GetCoordinateResetStep());
}
if constexpr(DstResetCoordinateAfterRun)
{
const auto dst_reset_step =
make_tensor_coordinate_step(dst_desc, GetCoordinateResetStep());
move_tensor_coordinate(dst_desc, dst_coord_, dst_reset_step);
move_tensor_coordinate(dst_desc, dst_coord_, GetCoordinateResetStep());
}
}
......@@ -179,13 +171,10 @@ struct ThreadwiseTensorSliceTransfer_v6r1r2
const Index& src_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = SrcResetCoordinateAfterRun
const auto adjusted_step = SrcResetCoordinateAfterRun
? src_slice_origin_step_idx
: src_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src_desc, adjusted_step_idx);
move_tensor_coordinate(src_desc, src_coord_, adjusted_step);
}
......@@ -194,13 +183,10 @@ struct ThreadwiseTensorSliceTransfer_v6r1r2
const Index& dst_slice_origin_step_idx)
{
// if dst coord was not reset by Run(), then need to adjust the step here
const auto adjusted_step_idx = DstResetCoordinateAfterRun
const auto adjusted_step = DstResetCoordinateAfterRun
? dst_slice_origin_step_idx
: dst_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(dst_desc, adjusted_step_idx);
move_tensor_coordinate(dst_desc, dst_coord_, adjusted_step);
}
......
include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r2.hpp
View file @ 0a7174ad
......@@ -147,38 +147,26 @@ struct ThreadwiseTensorSliceTransfer_v6r2
if constexpr(idx_1d.value != num_access - 1)
{
constexpr auto forward_step = SpaceFillingCurve::GetForwardStep(idx_1d);
move_tensor_coordinate(
src0_desc, src0_coord_, make_tensor_coordinate_step(src0_desc, forward_step));
move_tensor_coordinate(
src1_desc, src1_coord_, make_tensor_coordinate_step(src1_desc, forward_step));
move_tensor_coordinate(
dst_desc, dst_coord_, make_tensor_coordinate_step(dst_desc, forward_step));
move_tensor_coordinate(src0_desc, src0_coord_, forward_step);
move_tensor_coordinate(src1_desc, src1_coord_, forward_step);
move_tensor_coordinate(dst_desc, dst_coord_, forward_step);
}
});
// move coordinate back to slice origin (or not)
if constexpr(Src0ResetCoordinateAfterRun)
{
const auto src0_reset_step =
make_tensor_coordinate_step(src0_desc, GetCoordinateResetStep());
move_tensor_coordinate(src0_desc, src0_coord_, src0_reset_step);
move_tensor_coordinate(src0_desc, src0_coord_, GetCoordinateResetStep());
}
if constexpr(Src1ResetCoordinateAfterRun)
{
const auto src1_reset_step =
make_tensor_coordinate_step(src1_desc, GetCoordinateResetStep());
move_tensor_coordinate(src1_desc, src1_coord_, src1_reset_step);
move_tensor_coordinate(src1_desc, src1_coord_, GetCoordinateResetStep());
}
if constexpr(DstResetCoordinateAfterRun)
{
const auto dst_reset_step =
make_tensor_coordinate_step(dst_desc, GetCoordinateResetStep());
move_tensor_coordinate(dst_desc, dst_coord_, dst_reset_step);
move_tensor_coordinate(dst_desc, dst_coord_, GetCoordinateResetStep());
}
}
......@@ -210,13 +198,10 @@ struct ThreadwiseTensorSliceTransfer_v6r2
const Index& src0_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = Src0ResetCoordinateAfterRun
const auto adjusted_step = Src0ResetCoordinateAfterRun
? src0_slice_origin_step_idx
: src0_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src0_desc, adjusted_step_idx);
move_tensor_coordinate(src0_desc, src0_coord_, adjusted_step);
}
......@@ -225,13 +210,10 @@ struct ThreadwiseTensorSliceTransfer_v6r2
const Index& src1_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = Src1ResetCoordinateAfterRun
const auto adjusted_step = Src1ResetCoordinateAfterRun
? src1_slice_origin_step_idx
: src1_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src1_desc, adjusted_step_idx);
move_tensor_coordinate(src1_desc, src1_coord_, adjusted_step);
}
......@@ -240,13 +222,10 @@ struct ThreadwiseTensorSliceTransfer_v6r2
const Index& dst_slice_origin_step_idx)
{
// if dst coord was not reset by Run(), then need to adjust the step here
const auto adjusted_step_idx = DstResetCoordinateAfterRun
const auto adjusted_step = DstResetCoordinateAfterRun
? dst_slice_origin_step_idx
: dst_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(dst_desc, adjusted_step_idx);
move_tensor_coordinate(dst_desc, dst_coord_, adjusted_step);
}
......
include/ck/tensor_operation/gpu/thread/threadwise_tensor_slice_transfer_v6r3.hpp
View file @ 0a7174ad
......@@ -171,48 +171,32 @@ struct ThreadwiseTensorSliceTransfer_v6r3
if constexpr(idx_1d.value != num_access - 1)
{
constexpr auto forward_step = SpaceFillingCurve::GetForwardStep(idx_1d);
move_tensor_coordinate(
src0_desc, src0_coord_, make_tensor_coordinate_step(src0_desc, forward_step));
move_tensor_coordinate(
src1_desc, src1_coord_, make_tensor_coordinate_step(src1_desc, forward_step));
move_tensor_coordinate(
src2_desc, src2_coord_, make_tensor_coordinate_step(src2_desc, forward_step));
move_tensor_coordinate(
dst_desc, dst_coord_, make_tensor_coordinate_step(dst_desc, forward_step));
move_tensor_coordinate(src0_desc, src0_coord_, forward_step);
move_tensor_coordinate(src1_desc, src1_coord_, forward_step);
move_tensor_coordinate(src2_desc, src2_coord_, forward_step);
move_tensor_coordinate(dst_desc, dst_coord_, forward_step);
}
});
// move coordinate back to slice origin (or not)
if constexpr(Src0ResetCoordinateAfterRun)
{
const auto src0_reset_step =
make_tensor_coordinate_step(src0_desc, GetCoordinateResetStep());
move_tensor_coordinate(src0_desc, src0_coord_, src0_reset_step);
move_tensor_coordinate(src0_desc, src0_coord_, GetCoordinateResetStep());
}
if constexpr(Src1ResetCoordinateAfterRun)
{
const auto src1_reset_step =
make_tensor_coordinate_step(src1_desc, GetCoordinateResetStep());
move_tensor_coordinate(src1_desc, src1_coord_, src1_reset_step);
move_tensor_coordinate(src1_desc, src1_coord_, GetCoordinateResetStep());
}
if constexpr(Src2ResetCoordinateAfterRun)
{
const auto src2_reset_step =
make_tensor_coordinate_step(src2_desc, GetCoordinateResetStep());
move_tensor_coordinate(src2_desc, src2_coord_, src2_reset_step);
move_tensor_coordinate(src2_desc, src2_coord_, GetCoordinateResetStep());
}
if constexpr(DstResetCoordinateAfterRun)
{
const auto dst_reset_step =
make_tensor_coordinate_step(dst_desc, GetCoordinateResetStep());
move_tensor_coordinate(dst_desc, dst_coord_, dst_reset_step);
move_tensor_coordinate(dst_desc, dst_coord_, GetCoordinateResetStep());
}
}
......@@ -244,13 +228,10 @@ struct ThreadwiseTensorSliceTransfer_v6r3
const Index& src0_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = Src0ResetCoordinateAfterRun
const auto adjusted_step = Src0ResetCoordinateAfterRun
? src0_slice_origin_step_idx
: src0_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src0_desc, adjusted_step_idx);
move_tensor_coordinate(src0_desc, src0_coord_, adjusted_step);
}
......@@ -259,13 +240,10 @@ struct ThreadwiseTensorSliceTransfer_v6r3
const Index& src1_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = Src1ResetCoordinateAfterRun
const auto adjusted_step = Src1ResetCoordinateAfterRun
? src1_slice_origin_step_idx
: src1_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src1_desc, adjusted_step_idx);
move_tensor_coordinate(src1_desc, src1_coord_, adjusted_step);
}
......@@ -274,13 +252,10 @@ struct ThreadwiseTensorSliceTransfer_v6r3
const Index& src2_slice_origin_step_idx)
{
// if src coord was not reset by RunRead(), then need to adjust the step here
const auto adjusted_step_idx = Src2ResetCoordinateAfterRun
const auto adjusted_step = Src2ResetCoordinateAfterRun
? src2_slice_origin_step_idx
: src2_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(src2_desc, adjusted_step_idx);
move_tensor_coordinate(src2_desc, src2_coord_, adjusted_step);
}
......@@ -289,13 +264,10 @@ struct ThreadwiseTensorSliceTransfer_v6r3
const Index& dst_slice_origin_step_idx)
{
// if dst coord was not reset by Run(), then need to adjust the step here
const auto adjusted_step_idx = DstResetCoordinateAfterRun
const auto adjusted_step = DstResetCoordinateAfterRun
? dst_slice_origin_step_idx
: dst_slice_origin_step_idx + GetCoordinateResetStep();
// is it OK to construct a new step every time?
const auto adjusted_step = make_tensor_coordinate_step(dst_desc, adjusted_step_idx);
move_tensor_coordinate(dst_desc, dst_coord_, adjusted_step);
}
......
include/ck/tile_program/tile/tile_window_impl_static_distribution.hpp
View file @ 0a7174ad
......@@ -6,6 +6,7 @@
#include "ck/utility/common_header.hpp"
#include "ck/tensor_description/tensor_adaptor.hpp"
#include "ck/tensor_description/tensor_adaptor_coordinate.hpp"
#include "ck/tensor_description/tensor_space_filling_curve.hpp"
#include "ck/tile_program/tile/tile_distribution.hpp"
#include "ck/tile_program/tile/static_tile_distribution_helper.hpp"
......
include/ck/utility/array_multi_index.hpp
View file @ 0a7174ad
......@@ -75,4 +75,20 @@ __host__ __device__ constexpr auto operator*(const MultiIndex<NSize>& a, const T
return r;
}
// MultiIndex = index_t * MultiIndex
template <index_t NSize>
__host__ __device__ constexpr auto operator*(index_t a, const MultiIndex<NSize>& x)
{
MultiIndex<NSize> r;
static_for<0, NSize, 1>{}([&](auto i) { r(i) = a * x[i]; });
return r;
}
// MultiIndex = MultiIndex * index_t
template <index_t NSize>
__host__ __device__ constexpr auto operator*(const MultiIndex<NSize>& x, index_t a)
{
return a * x;
}
} // namespace ck
include/ck/utility/buffer_view_impl_global.hpp
View file @ 0a7174ad
......@@ -239,7 +239,7 @@ struct BufferView<AddressSpaceEnum::Global,
{
constexpr index_t t_per_x = scalar_per_x_vector / scalar_per_t_vector;
amd_buffer_atomic_add<remove_cvref_t<T>, t_per_x, Coherence>(
amd_buffer_atomic_add<remove_cvref_t<T>, t_per_x>(
x, p_data_, i, is_valid_element, buffer_size_);
}
else
......
include/ck/utility/container_helper.hpp
View file @ 0a7174ad
......@@ -471,7 +471,7 @@ __host__ __device__ constexpr auto sequence_to_tuple_of_number(Sequence<Is...>)
constexpr index_t tmp = Seq::At(i);
return Number<tmp>{};
},
Seq::Size());
Number<Seq::Size()>{});
}
} // namespace ck
include/ck/utility/data_type.hpp
View file @ 0a7174ad
......@@ -9,6 +9,7 @@
namespace ck {
using int64_t = long;
using bhalf_t = ushort;
using half_t = _Float16;
#ifdef CK_EXPERIMENTAL_BIT_INT_EXTENSION_INT4
......@@ -122,6 +123,13 @@ struct scalar_type<bhalf_t>
static constexpr index_t vector_size = 1;
};
template <>
struct scalar_type<int64_t>
{
using type = int64_t;
static constexpr index_t vector_size = 1;
};
template <>
struct scalar_type<int32_t>
{
......@@ -908,8 +916,6 @@ struct vector_type<T, 256>
}
};
using int64_t = long;
// fp64
using double2_t = typename vector_type<double, 2>::type;
using double4_t = typename vector_type<double, 4>::type;
......
include/ck/utility/magic_division.hpp
View file @ 0a7174ad
......@@ -4,10 +4,11 @@
#pragma once
#include "ck/ck.hpp"
#include "integral_constant.hpp"
#include "number.hpp"
#include "type.hpp"
#include "tuple.hpp"
#include "ck/utility/integral_constant.hpp"
#include "ck/utility/number.hpp"
#include "ck/utility/type.hpp"
#include "ck/utility/tuple.hpp"
#include "ck/utility/bit_cast.hpp"
namespace ck {
......
library/include/ck/library/tensor_operation_instance/add_device_operation_instance.hpp
View file @ 0a7174ad
......@@ -7,6 +7,7 @@
#include <type_traits>
#include "ck/utility/functional2.hpp"
#include "ck/utility/remove_cvref.hpp"
namespace ck {
namespace tensor_operation {
......
test/block_to_ctile_map/test_block_to_ctile_map.cpp
View file @ 0a7174ad
......@@ -6,6 +6,7 @@
#include <gtest/gtest.h>
#include "ck/ck.hpp"
#include "ck/tensor_description/tensor_descriptor_helper.hpp"
#include "ck/tensor_operation/gpu/grid/block_to_ctile_map.hpp"
using namespace ck;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment