Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ColossalAI
Commits
58356312
Commit
58356312
authored
Mar 31, 2022
by
coder-chin
Committed by
binmakeswell
Apr 06, 2022
Browse files
fix format (#564)
parent
e014144c
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
4 additions
and
3 deletions
+4
-3
colossalai/kernel/cuda_native/csrc/kernels/softmax_kernels.cu
...ssalai/kernel/cuda_native/csrc/kernels/softmax_kernels.cu
+4
-3
No files found.
colossalai/kernel/cuda_native/csrc/kernels/softmax_kernels.cu
View file @
58356312
...
@@ -120,7 +120,7 @@ __global__ void ker_attn_softmax(T *inp, const T *attn_mask, int from_len,
...
@@ -120,7 +120,7 @@ __global__ void ker_attn_softmax(T *inp, const T *attn_mask, int from_len,
BlockStore
(
ts_store
).
Store
(
inp
+
(
token_id
+
i
)
*
to_len
,
inp_val
[
i
],
BlockStore
(
ts_store
).
Store
(
inp
+
(
token_id
+
i
)
*
to_len
,
inp_val
[
i
],
to_len
);
to_len
);
}
}
}
// blockIdx.x
}
// blockIdx.x
}
}
template
<
typename
T
,
int
block_dim
,
int
ele_per_thread
>
template
<
typename
T
,
int
block_dim
,
int
ele_per_thread
>
...
@@ -198,7 +198,7 @@ __global__ void ker_attn_softmax_lt32(T *inp, const T *attn_mask, int from_len,
...
@@ -198,7 +198,7 @@ __global__ void ker_attn_softmax_lt32(T *inp, const T *attn_mask, int from_len,
BlockStore
(
ts_store
).
Store
(
inp
+
(
token_id
+
i
)
*
to_len
,
inp_val
[
i
],
BlockStore
(
ts_store
).
Store
(
inp
+
(
token_id
+
i
)
*
to_len
,
inp_val
[
i
],
to_len
);
to_len
);
}
}
}
// blockIdx.x
}
// blockIdx.x
}
}
/*
/*
...
@@ -304,7 +304,8 @@ __global__ void ker_attn_softmax_bw(T *grad, const T *inp, int softmax_length) {
...
@@ -304,7 +304,8 @@ __global__ void ker_attn_softmax_bw(T *grad, const T *inp, int softmax_length) {
cg
::
thread_block
b
=
cg
::
this_thread_block
();
cg
::
thread_block
b
=
cg
::
this_thread_block
();
cg
::
thread_block_tile
<
WARP_SIZE
>
g
=
cg
::
tiled_partition
<
WARP_SIZE
>
(
b
);
cg
::
thread_block_tile
<
WARP_SIZE
>
g
=
cg
::
tiled_partition
<
WARP_SIZE
>
(
b
);
for
(
int
i
=
1
;
i
<
WARP_SIZE
;
i
<<=
1
)
sum
+=
g
.
shfl_xor
(
sum
,
i
);
for
(
int
i
=
1
;
i
<
WARP_SIZE
;
i
<<=
1
)
sum
+=
g
.
shfl_xor
(
sum
,
i
);
#pragma unroll
#pragma unroll
for
(
int
i
=
0
;
i
<
ITERATIONS
;
++
i
)
{
for
(
int
i
=
0
;
i
<
ITERATIONS
;
++
i
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment