Skip to content

GitLab

"docs/git@developer.sourcefind.cn:change/sglang.git" did not exist on "11fffbc95a919a2446ae10fc33753d9951374fdf"
Commit 7f77ec65 authored by Davis King's avatar Davis King
Browse files

Made the batch normalization epsilon user settable rather than being hard coded.

parent b92b226c
Show whitespace changes
Inline Side-by-side
Showing with 162 additions and 73 deletions
dlib/dnn/cpu_dlib.cpp
View file @ 7f77ec65
...@@ -531,6 +531,7 @@ namespace dlib ...@@ -531,6 +531,7 @@ namespace dlib
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -546,7 +547,8 @@ namespace dlib ...@@ -546,7 +547,8 @@ namespace dlib
gamma.k() == src.k() && gamma.k() == src.k() &&
have_same_dimensions(gamma, beta) && have_same_dimensions(gamma, beta) &&
have_same_dimensions(gamma, running_means) && have_same_dimensions(gamma, running_means) &&
have_same_dimensions(gamma, running_variances), have_same_dimensions(gamma, running_variances) &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -565,7 +567,8 @@ namespace dlib ...@@ -565,7 +567,8 @@ namespace dlib
"\nrunning_variances.nc(): " << running_variances.nc() << "\nrunning_variances.nc(): " << running_variances.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
dest.copy_size(src); dest.copy_size(src);
...@@ -581,7 +584,7 @@ namespace dlib ...@@ -581,7 +584,7 @@ namespace dlib
{ {
for (long k = 0; k < num; ++k) for (long k = 0; k < num; ++k)
{ {
*d = g[k]*(*s - m[k])/std::sqrt(v[k]+dlib::tt::BATCH_NORM_EPS) + b[k]; *d = g[k]*(*s - m[k])/std::sqrt(v[k]+eps) + b[k];
++d; ++d;
++s; ++s;
} }
...@@ -589,6 +592,7 @@ namespace dlib ...@@ -589,6 +592,7 @@ namespace dlib
} }
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -609,7 +613,8 @@ namespace dlib ...@@ -609,7 +613,8 @@ namespace dlib
beta.num_samples() == 1 && beta.num_samples() == 1 &&
gamma.nr() == beta.nr() && beta.nr() == src.nr() && gamma.nr() == beta.nr() && beta.nr() == src.nr() &&
gamma.nc() == beta.nc() && beta.nc() == src.nc() && gamma.nc() == beta.nc() && beta.nc() == src.nc() &&
gamma.k() == beta.k() && beta.k() == src.k(), gamma.k() == beta.k() && beta.k() == src.k() &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -620,7 +625,8 @@ namespace dlib ...@@ -620,7 +625,8 @@ namespace dlib
"\nbeta.nc(): " << beta.nc() << "\nbeta.nc(): " << beta.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
dest.copy_size(src); dest.copy_size(src);
...@@ -662,7 +668,7 @@ namespace dlib ...@@ -662,7 +668,7 @@ namespace dlib
else else
rvar[i] = (1-averaging_factor)*rvar[i] + scale*averaging_factor*actual_var; rvar[i] = (1-averaging_factor)*rvar[i] + scale*averaging_factor*actual_var;
p_invstds[i] = 1.0f/std::sqrt(actual_var + dlib::tt::BATCH_NORM_EPS); p_invstds[i] = 1.0f/std::sqrt(actual_var + eps);
} }
p_src = src.host(); p_src = src.host();
...@@ -689,6 +695,7 @@ namespace dlib ...@@ -689,6 +695,7 @@ namespace dlib
} }
void batch_normalize_gradient ( void batch_normalize_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -709,6 +716,7 @@ namespace dlib ...@@ -709,6 +716,7 @@ namespace dlib
DLIB_CASSERT(num == beta_grad.size(),""); DLIB_CASSERT(num == beta_grad.size(),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),"");
DLIB_CASSERT(eps > 0,"");
beta_grad = 0; beta_grad = 0;
gamma_grad = 0; gamma_grad = 0;
...@@ -784,6 +792,7 @@ namespace dlib ...@@ -784,6 +792,7 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -799,7 +808,8 @@ namespace dlib ...@@ -799,7 +808,8 @@ namespace dlib
gamma.k() == src.k() && gamma.k() == src.k() &&
have_same_dimensions(gamma, beta) && have_same_dimensions(gamma, beta) &&
have_same_dimensions(gamma, running_means) && have_same_dimensions(gamma, running_means) &&
have_same_dimensions(gamma, running_variances), have_same_dimensions(gamma, running_variances) &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -818,7 +828,8 @@ namespace dlib ...@@ -818,7 +828,8 @@ namespace dlib
"\nrunning_variances.nc(): " << running_variances.nc() << "\nrunning_variances.nc(): " << running_variances.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
dest.copy_size(src); dest.copy_size(src);
...@@ -834,7 +845,7 @@ namespace dlib ...@@ -834,7 +845,7 @@ namespace dlib
{ {
for (long k = 0; k < src.k(); ++k) for (long k = 0; k < src.k(); ++k)
{ {
const float invstd = 1.0f/std::sqrt(v[k] + dlib::tt::BATCH_NORM_EPS); const float invstd = 1.0f/std::sqrt(v[k] + eps);
for (long j = 0; j < num; ++j) for (long j = 0; j < num; ++j)
{ {
*d = g[k]*(*s - m[k])*invstd + b[k]; *d = g[k]*(*s - m[k])*invstd + b[k];
...@@ -846,6 +857,7 @@ namespace dlib ...@@ -846,6 +857,7 @@ namespace dlib
} }
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -868,7 +880,8 @@ namespace dlib ...@@ -868,7 +880,8 @@ namespace dlib
beta.nr() == 1 && beta.nr() == 1 &&
gamma.nc() == 1 && gamma.nc() == 1 &&
beta.nc() == 1 && beta.nc() == 1 &&
gamma.k() == beta.k() && beta.k() == src.k(), gamma.k() == beta.k() && beta.k() == src.k() &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -879,7 +892,8 @@ namespace dlib ...@@ -879,7 +892,8 @@ namespace dlib
"\nbeta.nc(): " << beta.nc() << "\nbeta.nc(): " << beta.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
dest.copy_size(src); dest.copy_size(src);
...@@ -927,7 +941,7 @@ namespace dlib ...@@ -927,7 +941,7 @@ namespace dlib
else else
rvar[k] = (1-averaging_factor)*rvar[k] + scale*averaging_factor*actual_var; rvar[k] = (1-averaging_factor)*rvar[k] + scale*averaging_factor*actual_var;
p_invstds[k] = 1.0f/std::sqrt(actual_var + dlib::tt::BATCH_NORM_EPS); p_invstds[k] = 1.0f/std::sqrt(actual_var + eps);
} }
p_src = src.host(); p_src = src.host();
...@@ -955,6 +969,7 @@ namespace dlib ...@@ -955,6 +969,7 @@ namespace dlib
} }
void batch_normalize_conv_gradient( void batch_normalize_conv_gradient(
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -975,6 +990,7 @@ namespace dlib ...@@ -975,6 +990,7 @@ namespace dlib
DLIB_CASSERT(src.k() == beta_grad.size(),""); DLIB_CASSERT(src.k() == beta_grad.size(),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),"");
DLIB_CASSERT(eps > 0,"");
beta_grad = 0; beta_grad = 0;
gamma_grad = 0; gamma_grad = 0;
......
dlib/dnn/cpu_dlib.h
View file @ 7f77ec65
...@@ -131,6 +131,7 @@ namespace dlib ...@@ -131,6 +131,7 @@ namespace dlib
// ----------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -140,6 +141,7 @@ namespace dlib ...@@ -140,6 +141,7 @@ namespace dlib
); );
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -152,6 +154,7 @@ namespace dlib ...@@ -152,6 +154,7 @@ namespace dlib
); );
void batch_normalize_gradient ( void batch_normalize_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -163,6 +166,7 @@ namespace dlib ...@@ -163,6 +166,7 @@ namespace dlib
); );
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -172,6 +176,7 @@ namespace dlib ...@@ -172,6 +176,7 @@ namespace dlib
); );
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -184,6 +189,7 @@ namespace dlib ...@@ -184,6 +189,7 @@ namespace dlib
); );
void batch_normalize_conv_gradient ( void batch_normalize_conv_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
......
dlib/dnn/cudnn_dlibapi.cpp
View file @ 7f77ec65
...@@ -338,6 +338,7 @@ namespace dlib ...@@ -338,6 +338,7 @@ namespace dlib
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -353,7 +354,8 @@ namespace dlib ...@@ -353,7 +354,8 @@ namespace dlib
gamma.k() == src.k() && gamma.k() == src.k() &&
have_same_dimensions(gamma, beta) && have_same_dimensions(gamma, beta) &&
have_same_dimensions(gamma, running_means) && have_same_dimensions(gamma, running_means) &&
have_same_dimensions(gamma, running_variances), have_same_dimensions(gamma, running_variances) &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -372,7 +374,8 @@ namespace dlib ...@@ -372,7 +374,8 @@ namespace dlib
"\nrunning_variances.nc(): " << running_variances.nc() << "\nrunning_variances.nc(): " << running_variances.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
const float in_scale = 1; const float in_scale = 1;
const float out_scale = 0; const float out_scale = 0;
...@@ -393,10 +396,11 @@ namespace dlib ...@@ -393,10 +396,11 @@ namespace dlib
beta.device(), beta.device(),
running_means.device(), running_means.device(),
running_variances.device(), running_variances.device(),
dlib::tt::BATCH_NORM_EPS)); eps));
} }
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -417,7 +421,8 @@ namespace dlib ...@@ -417,7 +421,8 @@ namespace dlib
beta.num_samples() == 1 && beta.num_samples() == 1 &&
gamma.nr() == beta.nr() && beta.nr() == src.nr() && gamma.nr() == beta.nr() && beta.nr() == src.nr() &&
gamma.nc() == beta.nc() && beta.nc() == src.nc() && gamma.nc() == beta.nc() && beta.nc() == src.nc() &&
gamma.k() == beta.k() && beta.k() == src.k(), gamma.k() == beta.k() && beta.k() == src.k() &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -428,7 +433,8 @@ namespace dlib ...@@ -428,7 +433,8 @@ namespace dlib
"\nbeta.nc(): " << beta.nc() << "\nbeta.nc(): " << beta.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
const float in_scale = 1; const float in_scale = 1;
...@@ -455,12 +461,13 @@ namespace dlib ...@@ -455,12 +461,13 @@ namespace dlib
averaging_factor, averaging_factor,
running_means.device(), running_means.device(),
running_variances.device(), running_variances.device(),
dlib::tt::BATCH_NORM_EPS, eps,
means.device(), means.device(),
invstds.device())); invstds.device()));
} }
void batch_normalize_gradient( void batch_normalize_gradient(
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -480,6 +487,7 @@ namespace dlib ...@@ -480,6 +487,7 @@ namespace dlib
DLIB_CASSERT(num == beta_grad.size(),""); DLIB_CASSERT(num == beta_grad.size(),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),"");
DLIB_CASSERT(eps > 0,"");
const float in_scale = 1; const float in_scale = 1;
const float out_scale = 1; const float out_scale = 1;
...@@ -503,7 +511,7 @@ namespace dlib ...@@ -503,7 +511,7 @@ namespace dlib
gamma.device(), gamma.device(),
gamma_grad.device(), gamma_grad.device(),
beta_grad.device(), beta_grad.device(),
dlib::tt::BATCH_NORM_EPS, eps,
means.device(), means.device(),
invstds.device())); invstds.device()));
} }
...@@ -511,6 +519,7 @@ namespace dlib ...@@ -511,6 +519,7 @@ namespace dlib
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -526,7 +535,8 @@ namespace dlib ...@@ -526,7 +535,8 @@ namespace dlib
gamma.k() == src.k() && gamma.k() == src.k() &&
have_same_dimensions(gamma, beta) && have_same_dimensions(gamma, beta) &&
have_same_dimensions(gamma, running_means) && have_same_dimensions(gamma, running_means) &&
have_same_dimensions(gamma, running_variances), have_same_dimensions(gamma, running_variances) &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -545,7 +555,8 @@ namespace dlib ...@@ -545,7 +555,8 @@ namespace dlib
"\nrunning_variances.nc(): " << running_variances.nc() << "\nrunning_variances.nc(): " << running_variances.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
const float in_scale = 1; const float in_scale = 1;
const float out_scale = 0; const float out_scale = 0;
...@@ -566,10 +577,11 @@ namespace dlib ...@@ -566,10 +577,11 @@ namespace dlib
beta.device(), beta.device(),
running_means.device(), running_means.device(),
running_variances.device(), running_variances.device(),
dlib::tt::BATCH_NORM_EPS)); eps));
} }
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -592,7 +604,8 @@ namespace dlib ...@@ -592,7 +604,8 @@ namespace dlib
beta.nr() == 1 && beta.nr() == 1 &&
gamma.nc() == 1 && gamma.nc() == 1 &&
beta.nc() == 1 && beta.nc() == 1 &&
gamma.k() == beta.k() && beta.k() == src.k(), gamma.k() == beta.k() && beta.k() == src.k() &&
eps > 0,
"\ngamma.num_samples(): " << gamma.num_samples() << "\ngamma.num_samples(): " << gamma.num_samples() <<
"\ngamma.k(): " << gamma.k() << "\ngamma.k(): " << gamma.k() <<
"\ngamma.nr(): " << gamma.nr() << "\ngamma.nr(): " << gamma.nr() <<
...@@ -603,7 +616,8 @@ namespace dlib ...@@ -603,7 +616,8 @@ namespace dlib
"\nbeta.nc(): " << beta.nc() << "\nbeta.nc(): " << beta.nc() <<
"\nsrc.k(): " << src.k() << "\nsrc.k(): " << src.k() <<
"\nsrc.nr(): " << src.nr() << "\nsrc.nr(): " << src.nr() <<
"\nsrc.nc(): " << src.nc() "\nsrc.nc(): " << src.nc() <<
"\neps: " << eps
); );
const float in_scale = 1; const float in_scale = 1;
const float out_scale = 0; const float out_scale = 0;
...@@ -629,12 +643,13 @@ namespace dlib ...@@ -629,12 +643,13 @@ namespace dlib
averaging_factor, averaging_factor,
running_means.device(), running_means.device(),
running_variances.device(), running_variances.device(),
dlib::tt::BATCH_NORM_EPS, eps,
means.device(), means.device(),
invstds.device())); invstds.device()));
} }
void batch_normalize_conv_gradient( void batch_normalize_conv_gradient(
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -653,6 +668,7 @@ namespace dlib ...@@ -653,6 +668,7 @@ namespace dlib
DLIB_CASSERT(src.k() == beta_grad.size(),""); DLIB_CASSERT(src.k() == beta_grad.size(),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src),"");
DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),""); DLIB_CASSERT(have_same_dimensions(gradient_input, src_grad),"");
DLIB_CASSERT(eps > 0,"");
const float in_scale = 1; const float in_scale = 1;
const float out_scale = 1; const float out_scale = 1;
...@@ -676,7 +692,7 @@ namespace dlib ...@@ -676,7 +692,7 @@ namespace dlib
gamma.device(), gamma.device(),
gamma_grad.device(), gamma_grad.device(),
beta_grad.device(), beta_grad.device(),
dlib::tt::BATCH_NORM_EPS, eps,
means.device(), means.device(),
invstds.device())); invstds.device()));
} }
......
dlib/dnn/cudnn_dlibapi.h
View file @ 7f77ec65
...@@ -135,6 +135,7 @@ namespace dlib ...@@ -135,6 +135,7 @@ namespace dlib
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -144,6 +145,7 @@ namespace dlib ...@@ -144,6 +145,7 @@ namespace dlib
); );
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -156,6 +158,7 @@ namespace dlib ...@@ -156,6 +158,7 @@ namespace dlib
); );
void batch_normalize_gradient( void batch_normalize_gradient(
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -169,6 +172,7 @@ namespace dlib ...@@ -169,6 +172,7 @@ namespace dlib
// ------------------------------------------------------------------------------------ // ------------------------------------------------------------------------------------
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -178,6 +182,7 @@ namespace dlib ...@@ -178,6 +182,7 @@ namespace dlib
); );
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -190,6 +195,7 @@ namespace dlib ...@@ -190,6 +195,7 @@ namespace dlib
); );
void batch_normalize_conv_gradient( void batch_normalize_conv_gradient(
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
......
dlib/dnn/layers.h
View file @ 7f77ec65
...@@ -650,23 +650,30 @@ namespace dlib ...@@ -650,23 +650,30 @@ namespace dlib
FC_MODE = 1 FC_MODE = 1
}; };
const double DEFAULT_BATCH_NORM_EPS = 0.00001;
template < template <
layer_mode mode layer_mode mode
> >
class bn_ class bn_
{ {
public: public:
explicit bn_(unsigned long window_size) : explicit bn_(
unsigned long window_size,
double eps_ = DEFAULT_BATCH_NORM_EPS
) :
num_updates(0), num_updates(0),
running_stats_window_size(window_size), running_stats_window_size(window_size),
learning_rate_multiplier(1), learning_rate_multiplier(1),
weight_decay_multiplier(0) weight_decay_multiplier(0),
eps(eps_)
{} {}
bn_() : bn_(1000) {} bn_() : bn_(1000) {}
layer_mode get_mode() const { return mode; } layer_mode get_mode() const { return mode; }
unsigned long get_running_stats_window_size () const { return running_stats_window_size; } unsigned long get_running_stats_window_size () const { return running_stats_window_size; }
double get_eps() const { return eps; }
double get_learning_rate_multiplier () const { return learning_rate_multiplier; } double get_learning_rate_multiplier () const { return learning_rate_multiplier; }
double get_weight_decay_multiplier () const { return weight_decay_multiplier; } double get_weight_decay_multiplier () const { return weight_decay_multiplier; }
...@@ -713,16 +720,16 @@ namespace dlib ...@@ -713,16 +720,16 @@ namespace dlib
if (num_updates <running_stats_window_size) if (num_updates <running_stats_window_size)
++num_updates; ++num_updates;
if (mode == FC_MODE) if (mode == FC_MODE)
tt::batch_normalize(output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b); tt::batch_normalize(eps, output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b);
else else
tt::batch_normalize_conv(output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b); tt::batch_normalize_conv(eps, output, means, invstds, decay, running_means, running_variances, sub.get_output(), g, b);
} }
else // we are running in testing mode so we just linearly scale the input tensor. else // we are running in testing mode so we just linearly scale the input tensor.
{ {
if (mode == FC_MODE) if (mode == FC_MODE)
tt::batch_normalize_inference(output, sub.get_output(), g, b, running_means, running_variances); tt::batch_normalize_inference(eps, output, sub.get_output(), g, b, running_means, running_variances);
else else
tt::batch_normalize_conv_inference(output, sub.get_output(), g, b, running_means, running_variances); tt::batch_normalize_conv_inference(eps, output, sub.get_output(), g, b, running_means, running_variances);
} }
} }
...@@ -733,9 +740,9 @@ namespace dlib ...@@ -733,9 +740,9 @@ namespace dlib
auto g_grad = gamma(params_grad, 0); auto g_grad = gamma(params_grad, 0);
auto b_grad = beta(params_grad, gamma.size()); auto b_grad = beta(params_grad, gamma.size());
if (mode == FC_MODE) if (mode == FC_MODE)
tt::batch_normalize_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad ); tt::batch_normalize_gradient(eps, gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
else else
tt::batch_normalize_conv_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad ); tt::batch_normalize_conv_gradient(eps, gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
} }
const tensor& get_layer_params() const { return params; } const tensor& get_layer_params() const { return params; }
...@@ -758,6 +765,7 @@ namespace dlib ...@@ -758,6 +765,7 @@ namespace dlib
serialize(item.running_stats_window_size, out); serialize(item.running_stats_window_size, out);
serialize(item.learning_rate_multiplier, out); serialize(item.learning_rate_multiplier, out);
serialize(item.weight_decay_multiplier, out); serialize(item.weight_decay_multiplier, out);
serialize(item.eps, out);
} }
friend void deserialize(bn_& item, std::istream& in) friend void deserialize(bn_& item, std::istream& in)
...@@ -798,12 +806,13 @@ namespace dlib ...@@ -798,12 +806,13 @@ namespace dlib
// We also need to flip the running_variances around since the previous // We also need to flip the running_variances around since the previous
// format saved the inverse standard deviations instead of variances. // format saved the inverse standard deviations instead of variances.
item.running_variances = 1.0f/squared(mat(item.running_variances)) - tt::BATCH_NORM_EPS; item.running_variances = 1.0f/squared(mat(item.running_variances)) - DEFAULT_BATCH_NORM_EPS;
} }
else if (version == "bn_con2" || version == "bn_fc2") else if (version == "bn_con2" || version == "bn_fc2")
{ {
deserialize(item.learning_rate_multiplier, in); deserialize(item.learning_rate_multiplier, in);
deserialize(item.weight_decay_multiplier, in); deserialize(item.weight_decay_multiplier, in);
deserialize(item.eps, in);
} }
else else
{ {
...@@ -811,6 +820,8 @@ namespace dlib ...@@ -811,6 +820,8 @@ namespace dlib
// implicitly 1. // implicitly 1.
item.learning_rate_multiplier = 1; item.learning_rate_multiplier = 1;
item.weight_decay_multiplier = 1; item.weight_decay_multiplier = 1;
item.eps = DEFAULT_BATCH_NORM_EPS;
} }
} }
...@@ -820,6 +831,7 @@ namespace dlib ...@@ -820,6 +831,7 @@ namespace dlib
out << "bn_con "; out << "bn_con ";
else else
out << "bn_fc "; out << "bn_fc ";
out << " eps="<<item.eps;
out << " learning_rate_mult="<<item.learning_rate_multiplier; out << " learning_rate_mult="<<item.learning_rate_multiplier;
out << " weight_decay_mult="<<item.weight_decay_multiplier; out << " weight_decay_mult="<<item.weight_decay_multiplier;
return out; return out;
...@@ -837,6 +849,7 @@ namespace dlib ...@@ -837,6 +849,7 @@ namespace dlib
unsigned long running_stats_window_size; unsigned long running_stats_window_size;
double learning_rate_multiplier; double learning_rate_multiplier;
double weight_decay_multiplier; double weight_decay_multiplier;
double eps;
}; };
template <typename SUBNET> template <typename SUBNET>
...@@ -1273,7 +1286,7 @@ namespace dlib ...@@ -1273,7 +1286,7 @@ namespace dlib
auto sg = gamma(temp,0); auto sg = gamma(temp,0);
auto sb = beta(temp,gamma.size()); auto sb = beta(temp,gamma.size());
g = pointwise_multiply(mat(sg), 1.0f/sqrt(mat(item.running_variances)+tt::BATCH_NORM_EPS)); g = pointwise_multiply(mat(sg), 1.0f/sqrt(mat(item.running_variances)+item.get_eps()));
b = mat(sb) - pointwise_multiply(mat(g), mat(item.running_means)); b = mat(sb) - pointwise_multiply(mat(g), mat(item.running_means));
} }
......
dlib/dnn/layers_abstract.h
View file @ 7f77ec65
...@@ -818,6 +818,8 @@ namespace dlib ...@@ -818,6 +818,8 @@ namespace dlib
FC_MODE = 1 // fully connected mode FC_MODE = 1 // fully connected mode
}; };
const double DEFAULT_BATCH_NORM_EPS = 0.00001;
template < template <
layer_mode mode layer_mode mode
> >
...@@ -857,17 +859,22 @@ namespace dlib ...@@ -857,17 +859,22 @@ namespace dlib
- #get_running_stats_window_size() == 1000 - #get_running_stats_window_size() == 1000
- #get_learning_rate_multiplier() == 1 - #get_learning_rate_multiplier() == 1
- #get_weight_decay_multiplier() == 0 - #get_weight_decay_multiplier() == 0
- #get_eps() == tt::DEFAULT_BATCH_NORM_EPS
!*/ !*/
explicit bn_( explicit bn_(
unsigned long window_size unsigned long window_size,
double eps = tt::DEFAULT_BATCH_NORM_EPS
); );
/*! /*!
requires
- eps > 0
ensures ensures
- #get_mode() == mode - #get_mode() == mode
- #get_running_stats_window_size() == window_size - #get_running_stats_window_size() == window_size
- #get_learning_rate_multiplier() == 1 - #get_learning_rate_multiplier() == 1
- #get_weight_decay_multiplier() == 0 - #get_weight_decay_multiplier() == 0
- #get_eps() == eps
!*/ !*/
layer_mode get_mode( layer_mode get_mode(
...@@ -886,6 +893,15 @@ namespace dlib ...@@ -886,6 +893,15 @@ namespace dlib
normalization after a convolutional layer you should use CONV_MODE. normalization after a convolutional layer you should use CONV_MODE.
!*/ !*/
double get_eps(
) const;
/*!
ensures
- When doing batch normalization, we are dividing by the standard
deviation. This epsilon value returned by this function is added to the
variance to prevent the division from dividing by zero.
!*/
unsigned long get_running_stats_window_size ( unsigned long get_running_stats_window_size (
) const; ) const;
/*! /*!
......
dlib/dnn/tensor_tools.cpp
View file @ 7f77ec65
...@@ -337,6 +337,7 @@ namespace dlib { namespace tt ...@@ -337,6 +337,7 @@ namespace dlib { namespace tt
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -346,13 +347,14 @@ namespace dlib { namespace tt ...@@ -346,13 +347,14 @@ namespace dlib { namespace tt
) )
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize_inference(dest,src,gamma,beta,running_means,running_variances); cuda::batch_normalize_inference(eps,dest,src,gamma,beta,running_means,running_variances);
#else #else
cpu::batch_normalize_inference(dest,src,gamma,beta,running_means,running_variances); cpu::batch_normalize_inference(eps,dest,src,gamma,beta,running_means,running_variances);
#endif #endif
} }
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& vars, resizable_tensor& vars,
...@@ -365,13 +367,14 @@ namespace dlib { namespace tt ...@@ -365,13 +367,14 @@ namespace dlib { namespace tt
) )
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize(dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta); cuda::batch_normalize(eps,dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta);
#else #else
cpu::batch_normalize(dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta); cpu::batch_normalize(eps,dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta);
#endif #endif
} }
void batch_normalize_gradient ( void batch_normalize_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -384,15 +387,16 @@ namespace dlib { namespace tt ...@@ -384,15 +387,16 @@ namespace dlib { namespace tt
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); cuda::batch_normalize_gradient(eps,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
#else #else
cpu::batch_normalize_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); cpu::batch_normalize_gradient(eps,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
#endif #endif
} }
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -402,13 +406,14 @@ namespace dlib { namespace tt ...@@ -402,13 +406,14 @@ namespace dlib { namespace tt
) )
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_inference(dest,src,gamma,beta,running_means,running_variances); cuda::batch_normalize_conv_inference(eps,dest,src,gamma,beta,running_means,running_variances);
#else #else
cpu::batch_normalize_conv_inference(dest,src,gamma,beta,running_means,running_variances); cpu::batch_normalize_conv_inference(eps,dest,src,gamma,beta,running_means,running_variances);
#endif #endif
} }
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& vars, resizable_tensor& vars,
...@@ -421,13 +426,14 @@ namespace dlib { namespace tt ...@@ -421,13 +426,14 @@ namespace dlib { namespace tt
) )
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv(dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta); cuda::batch_normalize_conv(eps,dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta);
#else #else
cpu::batch_normalize_conv(dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta); cpu::batch_normalize_conv(eps,dest,means,vars,averaging_factor,running_means,running_variances,src,gamma,beta);
#endif #endif
} }
void batch_normalize_conv_gradient ( void batch_normalize_conv_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -440,9 +446,9 @@ namespace dlib { namespace tt ...@@ -440,9 +446,9 @@ namespace dlib { namespace tt
{ {
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
cuda::batch_normalize_conv_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); cuda::batch_normalize_conv_gradient(eps,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
#else #else
cpu::batch_normalize_conv_gradient(gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad); cpu::batch_normalize_conv_gradient(eps,gradient_input, means, invstds, src, gamma, src_grad, gamma_grad, beta_grad);
#endif #endif
} }
......
dlib/dnn/tensor_tools.h
View file @ 7f77ec65
...@@ -370,9 +370,8 @@ namespace dlib { namespace tt ...@@ -370,9 +370,8 @@ namespace dlib { namespace tt
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
const double BATCH_NORM_EPS = 0.00001;
void batch_normalize_inference ( void batch_normalize_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -382,6 +381,7 @@ namespace dlib { namespace tt ...@@ -382,6 +381,7 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- gamma.num_samples() == 1 - gamma.num_samples() == 1
- gamma.nr() == src.nr() - gamma.nr() == src.nr()
- gamma.nc() == src.nc() - gamma.nc() == src.nc()
...@@ -393,11 +393,12 @@ namespace dlib { namespace tt ...@@ -393,11 +393,12 @@ namespace dlib { namespace tt
- Linearly transforms src as a call to batch_normalize() would if src had means - Linearly transforms src as a call to batch_normalize() would if src had means
and variances as given by running_means and running_variances. That is, this and variances as given by running_means and running_variances. That is, this
function performs: function performs:
dest = gamma*(src-running_means)/sqrt(running_variances+BATCH_NORM_EPS) + beta dest = gamma*(src-running_means)/sqrt(running_variances+eps) + beta
Note that it does it in a pointwise fashion over the samples in src. Note that it does it in a pointwise fashion over the samples in src.
!*/ !*/
void batch_normalize ( void batch_normalize (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -410,6 +411,7 @@ namespace dlib { namespace tt ...@@ -410,6 +411,7 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- src.num_samples() > 1 - src.num_samples() > 1
- gamma.num_samples() == 1 - gamma.num_samples() == 1
- beta.num_samples() == 1 - beta.num_samples() == 1
...@@ -435,6 +437,7 @@ namespace dlib { namespace tt ...@@ -435,6 +437,7 @@ namespace dlib { namespace tt
!*/ !*/
void batch_normalize_gradient ( void batch_normalize_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -446,8 +449,9 @@ namespace dlib { namespace tt ...@@ -446,8 +449,9 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- invstds and means should be the output of a call to - invstds and means should be the output of a call to
batch_normalize(dest,means,invstds,src,gamma,beta) batch_normalize(eps,dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true - have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true - have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1 - src.num_samples() > 1
...@@ -461,7 +465,7 @@ namespace dlib { namespace tt ...@@ -461,7 +465,7 @@ namespace dlib { namespace tt
- have_same_dimensions(invstds, gamma) == true - have_same_dimensions(invstds, gamma) == true
ensures ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of - Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize(dest,means,invstds,src,gamma,beta)) batch_normalize(eps,dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad. - Adds the gradient of f() with respect to src to #src_grad.
- Assigns the gradient of f() with respect to gamma to #gamma_grad. - Assigns the gradient of f() with respect to gamma to #gamma_grad.
- Assigns the gradient of f() with respect to beta to #beta_grad. - Assigns the gradient of f() with respect to beta to #beta_grad.
...@@ -470,6 +474,7 @@ namespace dlib { namespace tt ...@@ -470,6 +474,7 @@ namespace dlib { namespace tt
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
void batch_normalize_conv_inference ( void batch_normalize_conv_inference (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
const tensor& src, const tensor& src,
const tensor& gamma, const tensor& gamma,
...@@ -479,6 +484,7 @@ namespace dlib { namespace tt ...@@ -479,6 +484,7 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- gamma.num_samples() == 1 - gamma.num_samples() == 1
- gamma.nr() == 1 - gamma.nr() == 1
- gamma.nc() == 1 - gamma.nc() == 1
...@@ -490,12 +496,13 @@ namespace dlib { namespace tt ...@@ -490,12 +496,13 @@ namespace dlib { namespace tt
- Linearly transforms src as a call to batch_normalize_conv() would if src had - Linearly transforms src as a call to batch_normalize_conv() would if src had
means and variances as given by running_means and running_variances. That means and variances as given by running_means and running_variances. That
is, this function performs: is, this function performs:
dest = gamma*(src-running_means)/sqrt(running_variances+BATCH_NORM_EPS) + beta dest = gamma*(src-running_means)/sqrt(running_variances+eps) + beta
Note that it does this in a pointwise fashion over the samples, rows, and Note that it does this in a pointwise fashion over the samples, rows, and
columns in src. columns in src.
!*/ !*/
void batch_normalize_conv ( void batch_normalize_conv (
const double eps,
resizable_tensor& dest, resizable_tensor& dest,
resizable_tensor& means, resizable_tensor& means,
resizable_tensor& invstds, resizable_tensor& invstds,
...@@ -508,6 +515,7 @@ namespace dlib { namespace tt ...@@ -508,6 +515,7 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- src.num_samples() > 1 - src.num_samples() > 1
- gamma.num_samples()==gamma.nr()==gamma.nc() == 1 - gamma.num_samples()==gamma.nr()==gamma.nc() == 1
- beta.num_samples() ==beta.nr() ==gamma.nc() == 1 - beta.num_samples() ==beta.nr() ==gamma.nc() == 1
...@@ -529,6 +537,7 @@ namespace dlib { namespace tt ...@@ -529,6 +537,7 @@ namespace dlib { namespace tt
!*/ !*/
void batch_normalize_conv_gradient ( void batch_normalize_conv_gradient (
const double eps,
const tensor& gradient_input, const tensor& gradient_input,
const tensor& means, const tensor& means,
const tensor& invstds, const tensor& invstds,
...@@ -540,8 +549,9 @@ namespace dlib { namespace tt ...@@ -540,8 +549,9 @@ namespace dlib { namespace tt
); );
/*! /*!
requires requires
- eps > 0
- invstds and means should be the output of a call to - invstds and means should be the output of a call to
batch_normalize_conv(dest,means,invstds,src,gamma,beta) batch_normalize_conv(eps,dest,means,invstds,src,gamma,beta)
- have_same_dimensions(gradient_input, src) == true - have_same_dimensions(gradient_input, src) == true
- have_same_dimensions(src, src_grad) == true - have_same_dimensions(src, src_grad) == true
- src.num_samples() > 1 - src.num_samples() > 1
...@@ -553,7 +563,7 @@ namespace dlib { namespace tt ...@@ -553,7 +563,7 @@ namespace dlib { namespace tt
- have_same_dimensions(invstds, gamma) == true - have_same_dimensions(invstds, gamma) == true
ensures ensures
- Let f(src,gamma,beta) == dot(gradient_input, dest output of - Let f(src,gamma,beta) == dot(gradient_input, dest output of
batch_normalize_conv(dest,means,invstds,src,gamma,beta)) batch_normalize_conv(eps,dest,means,invstds,src,gamma,beta))
- Adds the gradient of f() with respect to src to #src_grad. - Adds the gradient of f() with respect to src to #src_grad.
- Assigns the gradient of f() with respect to gamma to #gamma_grad. - Assigns the gradient of f() with respect to gamma to #gamma_grad.
- Assigns the gradient of f() with respect to beta to #beta_grad. - Assigns the gradient of f() with respect to beta to #beta_grad.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment