Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
MIGraphX
Commits
6c41008a
Commit
6c41008a
authored
Dec 08, 2023
by
charlie
Browse files
formatting
parent
153276ce
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
22 additions
and
22 deletions
+22
-22
src/rewrite_gelu.cpp
src/rewrite_gelu.cpp
+22
-22
No files found.
src/rewrite_gelu.cpp
View file @
6c41008a
...
@@ -33,8 +33,8 @@ namespace migraphx {
...
@@ -33,8 +33,8 @@ namespace migraphx {
inline
namespace
MIGRAPHX_INLINE_NS
{
inline
namespace
MIGRAPHX_INLINE_NS
{
/**
/**
* Finds GELU blocks using the Gaussian distribution and replaces them with the sigmoid
approximation
* Finds GELU blocks using the Gaussian distribution and replaces them with the sigmoid
* if the data type is fp16.
*
approximation
if the data type is fp16.
*/
*/
struct
find_gelu_erf
struct
find_gelu_erf
{
{
...
@@ -70,31 +70,31 @@ struct find_tanh_fast_gelu
...
@@ -70,31 +70,31 @@ struct find_tanh_fast_gelu
void
apply
(
module
&
m
,
const
match
::
matcher_result
&
r
)
const
void
apply
(
module
&
m
,
const
match
::
matcher_result
&
r
)
const
{
{
auto
ins
=
r
.
result
;
auto
ins
=
r
.
result
;
auto
x
=
r
.
instructions
[
"x"
];
auto
x
=
r
.
instructions
[
"x"
];
auto
sqrt_2_rpi
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
0.7978845608028653558798921198687637369517172623298693153318516593
}});
auto
sqrt_2_rpi
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
0.7978845608028653558798921198687637369517172623298693153318516593
}});
auto
fit_const
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
0.044715
f
}});
auto
fit_const
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
0.044715
f
}});
auto
one
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
1.0
f
}});
auto
one
=
m
.
add_literal
(
literal
{
shape
{
x
->
get_shape
().
type
()},
{
1.0
f
}});
auto
xb
=
insert_common_op
(
m
,
ins
,
make_op
(
"mul"
),
{
x
,
sqrt_2_rpi
});
auto
xb
=
insert_common_op
(
m
,
ins
,
make_op
(
"mul"
),
{
x
,
sqrt_2_rpi
});
auto
a
=
insert_common_op
(
m
,
ins
,
make_op
(
"mul"
),
{
xb
,
fit_const
});
auto
a
=
insert_common_op
(
m
,
ins
,
make_op
(
"mul"
),
{
xb
,
fit_const
});
auto
b
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
a
,
x
);
auto
b
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
a
,
x
);
auto
c
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
b
,
x
);
auto
c
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
b
,
x
);
auto
u
=
m
.
insert_instruction
(
ins
,
make_op
(
"add"
),
c
,
xb
);
auto
u
=
m
.
insert_instruction
(
ins
,
make_op
(
"add"
),
c
,
xb
);
auto
neg_u
=
m
.
insert_instruction
(
ins
,
make_op
(
"neg"
),
u
);
auto
neg_u
=
m
.
insert_instruction
(
ins
,
make_op
(
"neg"
),
u
);
auto
d
=
m
.
insert_instruction
(
ins
,
make_op
(
"sub"
),
neg_u
,
u
);
auto
d
=
m
.
insert_instruction
(
ins
,
make_op
(
"sub"
),
neg_u
,
u
);
auto
emu
=
m
.
insert_instruction
(
ins
,
make_op
(
"exp"
),
d
);
auto
emu
=
m
.
insert_instruction
(
ins
,
make_op
(
"exp"
),
d
);
auto
e
=
insert_common_op
(
m
,
ins
,
make_op
(
"add"
),
{
one
,
emu
});
auto
e
=
insert_common_op
(
m
,
ins
,
make_op
(
"add"
),
{
one
,
emu
});
auto
cdf
=
insert_common_op
(
m
,
ins
,
make_op
(
"div"
),
{
one
,
e
});
auto
cdf
=
insert_common_op
(
m
,
ins
,
make_op
(
"div"
),
{
one
,
e
});
auto
y
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
x
,
cdf
);
auto
y
=
m
.
insert_instruction
(
ins
,
make_op
(
"mul"
),
x
,
cdf
);
m
.
replace_instruction
(
ins
,
y
);
m
.
replace_instruction
(
ins
,
y
);
}
}
};
};
void
rewrite_gelu
::
apply
(
module
&
m
)
const
{
void
rewrite_gelu
::
apply
(
module
&
m
)
const
match
::
find_matches
(
m
,
{
find_gelu_erf
{},
match
::
find_matches
(
m
,
find_gelu_erf
{},
find_tanh_fast_gelu
{});
find_tanh_fast_gelu
{}
);
}
}
}
// namespace MIGRAPHX_INLINE_NS
}
// namespace MIGRAPHX_INLINE_NS
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment