Skip to content

GitLab

"docs/git@developer.sourcefind.cn:OpenDAS/torchaudio.git" did not exist on "928248d706ead8250cb190878b04a4d38cc67a4d"
Commit 93ab80c7 authored by Davis King's avatar Davis King
Browse files

Made the affine_ layer support being constructed from bn_ layers. Also added 

unit tests for the routines supporting this feature.
parent 669a1e17
Hide whitespace changes
Inline Side-by-side
Showing with 309 additions and 76 deletions
dlib/dnn/layers.h
View file @ 93ab80c7
...@@ -418,27 +418,31 @@ namespace dlib ...@@ -418,27 +418,31 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
enum batch_normalization_mode enum layer_mode
{ {
BATCH_NORM_CONV = 0, CONV_MODE = 0,
BATCH_NORM_FC = 1 FC_MODE = 1
}; };
class bn_ class bn_
{ {
public: public:
bn_() : num_updates(0), running_stats_window_size(1000), mode(BATCH_NORM_FC) bn_() : num_updates(0), running_stats_window_size(1000), mode(FC_MODE)
{} {}
explicit bn_(batch_normalization_mode mode_) : num_updates(0), running_stats_window_size(1000), mode(mode_) explicit bn_(layer_mode mode_) : num_updates(0), running_stats_window_size(1000), mode(mode_)
{} {}
batch_normalization_mode get_mode() const { return mode; } bn_(layer_mode mode_, unsigned long window_size) : num_updates(0), running_stats_window_size(window_size), mode(mode_)
{}
layer_mode get_mode() const { return mode; }
unsigned long get_running_stats_window_size () const { return running_stats_window_size; }
template <typename SUBNET> template <typename SUBNET>
void setup (const SUBNET& sub) void setup (const SUBNET& sub)
{ {
if (mode == BATCH_NORM_FC) if (mode == FC_MODE)
{ {
gamma = alias_tensor(1, gamma = alias_tensor(1,
sub.get_output().k(), sub.get_output().k(),
...@@ -473,14 +477,14 @@ namespace dlib ...@@ -473,14 +477,14 @@ namespace dlib
const double decay = 1.0 - num_updates/(num_updates+1.0); const double decay = 1.0 - num_updates/(num_updates+1.0);
if (num_updates <running_stats_window_size) if (num_updates <running_stats_window_size)
++num_updates; ++num_updates;
if (mode == BATCH_NORM_FC) if (mode == FC_MODE)
tt::batch_normalize(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b); tt::batch_normalize(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b);
else else
tt::batch_normalize_conv(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b); tt::batch_normalize_conv(output, means, invstds, decay, running_means, running_invstds, sub.get_output(), g, b);
} }
else // we are running in testing mode so we just linearly scale the input tensor. else // we are running in testing mode so we just linearly scale the input tensor.
{ {
if (mode == BATCH_NORM_FC) if (mode == FC_MODE)
tt::batch_normalize_inference(output, sub.get_output(), g, b, running_means, running_invstds); tt::batch_normalize_inference(output, sub.get_output(), g, b, running_means, running_invstds);
else else
tt::batch_normalize_conv_inference(output, sub.get_output(), g, b, running_means, running_invstds); tt::batch_normalize_conv_inference(output, sub.get_output(), g, b, running_means, running_invstds);
...@@ -493,7 +497,7 @@ namespace dlib ...@@ -493,7 +497,7 @@ namespace dlib
auto g = gamma(params,0); auto g = gamma(params,0);
auto g_grad = gamma(params_grad, 0); auto g_grad = gamma(params_grad, 0);
auto b_grad = beta(params_grad, gamma.size()); auto b_grad = beta(params_grad, gamma.size());
if (mode == BATCH_NORM_FC) if (mode == FC_MODE)
tt::batch_normalize_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad ); tt::batch_normalize_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
else else
tt::batch_normalize_conv_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad ); tt::batch_normalize_conv_gradient(gradient_input, means, invstds, sub.get_output(), g, sub.get_gradient_input(), g_grad, b_grad );
...@@ -534,18 +538,20 @@ namespace dlib ...@@ -534,18 +538,20 @@ namespace dlib
deserialize(item.running_stats_window_size, in); deserialize(item.running_stats_window_size, in);
int mode; int mode;
deserialize(mode, in); deserialize(mode, in);
item.mode = (batch_normalization_mode)mode; item.mode = (layer_mode)mode;
} }
private: private:
friend class affine_;
resizable_tensor params; resizable_tensor params;
alias_tensor gamma, beta; alias_tensor gamma, beta;
resizable_tensor means, running_means; resizable_tensor means, running_means;
resizable_tensor invstds, running_invstds; resizable_tensor invstds, running_invstds;
unsigned long num_updates; unsigned long num_updates;
unsigned long running_stats_window_size; unsigned long running_stats_window_size;
batch_normalization_mode mode; layer_mode mode;
}; };
template <typename SUBNET> template <typename SUBNET>
...@@ -770,17 +776,53 @@ namespace dlib ...@@ -770,17 +776,53 @@ namespace dlib
{ {
public: public:
affine_( affine_(
) ) : mode(FC_MODE)
{
}
explicit affine_(
layer_mode mode_
) : mode(mode_)
{ {
} }
affine_(
const bn_& item
)
{
gamma = item.gamma;
beta = item.beta;
mode = item.mode;
params.copy_size(item.params);
auto g = gamma(params,0);
auto b = beta(params,gamma.size());
resizable_tensor temp(item.params);
auto sg = gamma(temp,0);
auto sb = beta(temp,gamma.size());
g = pointwise_multiply(mat(sg), mat(item.running_invstds));
b = mat(sb) - pointwise_multiply(mat(g), mat(item.running_means));
}
layer_mode get_mode() const { return mode; }
template <typename SUBNET> template <typename SUBNET>
void setup (const SUBNET& sub) void setup (const SUBNET& sub)
{ {
gamma = alias_tensor(1, if (mode == FC_MODE)
sub.get_output().k(), {
sub.get_output().nr(), gamma = alias_tensor(1,
sub.get_output().nc()); sub.get_output().k(),
sub.get_output().nr(),
sub.get_output().nc());
}
else
{
gamma = alias_tensor(1, sub.get_output().k());
}
beta = gamma; beta = gamma;
params.set_size(gamma.size()+beta.size()); params.set_size(gamma.size()+beta.size());
...@@ -793,7 +835,10 @@ namespace dlib ...@@ -793,7 +835,10 @@ namespace dlib
{ {
auto g = gamma(params,0); auto g = gamma(params,0);
auto b = beta(params,gamma.size()); auto b = beta(params,gamma.size());
tt::affine_transform(output, input, g, b); if (mode == FC_MODE)
tt::affine_transform(output, input, g, b);
else
tt::affine_transform_conv(output, input, g, b);
} }
void backward_inplace( void backward_inplace(
...@@ -809,10 +854,18 @@ namespace dlib ...@@ -809,10 +854,18 @@ namespace dlib
auto b_grad = beta(params_grad,gamma.size()); auto b_grad = beta(params_grad,gamma.size());
// We are computing the gradient of dot(gradient_input, computed_output*g + b) // We are computing the gradient of dot(gradient_input, computed_output*g + b)
tt::multiply(data_grad, gradient_input, g); if (mode == FC_MODE)
{
tt::multiply(g_grad, gradient_input, computed_output); tt::multiply(data_grad, gradient_input, g);
tt::assign_bias_gradient(b_grad, gradient_input); tt::multiply(g_grad, gradient_input, computed_output);
tt::assign_bias_gradient(b_grad, gradient_input);
}
else
{
tt::multiply_conv(data_grad, gradient_input, g);
tt::multiply_conv(g_grad, gradient_input, computed_output);
tt::assign_conv_bias_gradient(b_grad, gradient_input);
}
} }
const tensor& get_layer_params() const { return params; } const tensor& get_layer_params() const { return params; }
...@@ -824,6 +877,7 @@ namespace dlib ...@@ -824,6 +877,7 @@ namespace dlib
serialize(item.params, out); serialize(item.params, out);
serialize(item.gamma, out); serialize(item.gamma, out);
serialize(item.beta, out); serialize(item.beta, out);
serialize((int)item.mode, out);
} }
friend void deserialize(affine_& item, std::istream& in) friend void deserialize(affine_& item, std::istream& in)
...@@ -835,11 +889,15 @@ namespace dlib ...@@ -835,11 +889,15 @@ namespace dlib
deserialize(item.params, in); deserialize(item.params, in);
deserialize(item.gamma, in); deserialize(item.gamma, in);
deserialize(item.beta, in); deserialize(item.beta, in);
int mode;
deserialize(mode, in);
item.mode = (layer_mode)mode;
} }
private: private:
resizable_tensor params; resizable_tensor params;
alias_tensor gamma, beta; alias_tensor gamma, beta;
layer_mode mode;
}; };
template <typename SUBNET> template <typename SUBNET>
......
dlib/dnn/layers_abstract.h
View file @ 93ab80c7
...@@ -552,29 +552,89 @@ namespace dlib ...@@ -552,29 +552,89 @@ namespace dlib
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
class affine_ enum layer_mode
{
CONV_MODE = 0, // convolutional mode
FC_MODE = 1 // fully connected mode
};
class bn_
{ {
/*! /*!
WHAT THIS OBJECT REPRESENTS WHAT THIS OBJECT REPRESENTS
This is an implementation of the EXAMPLE_LAYER_ interface defined above. This is an implementation of the EXAMPLE_LAYER_ interface defined above.
In particular, it applies a simple pointwise linear transformation to an In particular, it defines a batch normalization layer that implements the
input tensor. You can think of it as having two parameter tensors, A and method described in the paper:
B, that each have the same dimensionality as the input tensor (except their Batch Normalization: Accelerating Deep Network Training by Reducing
num_samples() dimensions are 1). If the input tensor is called INPUT Internal Covariate Shift by Sergey Ioffe and Christian Szegedy
then the output of this layer is simply:
A*INPUT+B In particular, this layer produces output tensors with the same
where all operations are performed element wise and each sample in the dimensionality as the input tensors, except that the mean and variances of
INPUT tensor is processed separately. the elements have been standardized to 0 and 1 respectively.
It should also be noted that when tensors with a num_samples() dimension of
1 are passed to this layer it doesn't perform batch normalization.
Instead, it runs in "inference mode" where the learned linear normalizing
transformation is used to transform the tensor.
Finally, after you finish training a batch normalized network, it is a good
idea to replace each bn_ layer with an affine_ layer because the affine_
layer is faster and will never surprise you by performing batch
normalization on tensors that have a num_samples() dimension > 1. This allows
you to run large mini-batches of samples through your final network without
batch normalization executing at all.
!*/ !*/
public: public:
bn_(
);
/*!
ensures
- #get_mode() == FC_MODE
- get_running_stats_window_size() == 1000
!*/
affine_( explicit bn_(
layer_mode mode
); );
/*!
ensures
- #get_mode() == mode
- get_running_stats_window_size() == 1000
!*/
layer_mode get_mode(
) const;
/*!
ensures
- returns the mode of this layer, either CONV_MODE or FC_MODE.
If the mode is FC_MODE then the normalization is applied across the
samples in a tensor (i.e. k()*nr()*nc() different things will be
normalized). Otherwise, normalization is applied across everything
except for the k() dimension, resulting in there being only k()
normalization equations that are applied spatially over the tensor.
Therefore, if you are putting batch normalization after a fully connected
layer you should use FC_MODE. Otherwise, if you are putting batch
normalization after a convolutional layer you should use CONV_MODE.
!*/
unsigned long get_running_stats_window_size (
) const;
/*!
ensures
- Just as recommended in the batch normalization paper, this object keeps a
running average of the mean and standard deviations of the features.
These averages are used during "inference mode" so you can run a single
object through a batch normalized network. They are also what is used to
initialize an affine_ layer that is constructed from a bn_ layer. This
function returns the effective number of recent samples used to compute
the running average.
!*/
template <typename SUBNET> void setup (const SUBNET& sub); template <typename SUBNET> void setup (const SUBNET& sub);
void forward_inplace(const tensor& input, tensor& output); template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output);
void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad);
const tensor& get_layer_params() const; const tensor& get_layer_params() const;
tensor& get_layer_params(); tensor& get_layer_params();
/*! /*!
...@@ -582,73 +642,83 @@ namespace dlib ...@@ -582,73 +642,83 @@ namespace dlib
!*/ !*/
}; };
void serialize(const affine_& item, std::ostream& out); void serialize(const bn_& item, std::ostream& out);
void deserialize(affine_& item, std::istream& in); void deserialize(bn_& item, std::istream& in);
/*! /*!
provides serialization support provides serialization support
!*/ !*/
template <typename SUBNET> template <typename SUBNET>
using affine = add_layer<affine_, SUBNET>; using bn = add_layer<bn_, SUBNET>;
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
enum batch_normalization_mode class affine_
{
BATCH_NORM_CONV = 0,
BATCH_NORM_FC = 1
};
class bn_
{ {
/*! /*!
WHAT THIS OBJECT REPRESENTS WHAT THIS OBJECT REPRESENTS
This is an implementation of the EXAMPLE_LAYER_ interface defined above. This is an implementation of the EXAMPLE_LAYER_ interface defined above.
In particular, it defines a batch normalization layer that implements the In particular, it applies a simple pointwise linear transformation to an
method described in the paper: input tensor. You can think of it as having two parameter tensors, A and
Batch Normalization: Accelerating Deep Network Training by Reducing B. If the input tensor is called INPUT then the output of this layer is:
Internal Covariate Shift by Sergey Ioffe and Christian Szegedy A*INPUT+B
where all operations are performed element wise and each sample in the
In particular, this layer produces output tensors with the same INPUT tensor is processed separately.
dimensionality as the input tensors, except that the mean and variances of
the elements have been standardized. Moreover, this object has two modes that effect the dimensionalities of A
and B and how they are applied to compute A*INPUT+B. If
get_mode()==FC_MODE then A and B each have the same dimensionality as the
input tensor, except their num_samples() dimensions are 1. If
get_mode()==CONV_MODE then A and B have all their dimensions set to 1
except for k(), which is equal to INPUT.k().
In either case, the computation of A*INPUT+B is performed pointwise over all
the elements of INPUT using either:
OUTPUT(n,k,r,c) == A(1,k,r,c)*INPUT(n,k,r,c)+B(1,k,r,c)
or
OUTPUT(n,k,r,c) == A(1,k,1,1)*INPUT(n,k,r,c)+B(1,k,1,1)
as appropriate.
!*/ !*/
public: public:
bn_(
affine_(
); );
/*! /*!
ensures ensures
- #get_mode() == BATCH_NORM_FC - #get_mode() == FC_MODE
!*/ !*/
explicit bn_( affine_(
batch_normalization_mode mode const bn_& layer
);
/*!
ensures
- Constructs affine_ so that it performs the same transformation as the
supplied batch normalization layer. You would want to do this after you
finish training a network with bn_ layers because the affine_ layer will
execute faster.
- #get_mode() == layer.get_mode()
!*/
explicit affine_(
layer_mode mode
); );
/*! /*!
ensures ensures
- #get_mode() == mode - #get_mode() == mode
!*/ !*/
batch_normalization_mode get_mode( layer_mode get_mode(
) const; ) const;
/*! /*!
ensures ensures
- returns the mode of this layer, either BATCH_NORM_CONV or BATCH_NORM_FC. - returns the mode of this layer, either CONV_MODE or FC_MODE.
If the mode is BATCH_NORM_FC then the normalization is applied across the
samples in a tensor (i.e. k()*nr()*nc() different things will be
normalized). Otherwise, normalization is applied across everything
except for the k() dimension, resulting in there being only k()
normalization equations that are applied spatially over the tensor.
Therefore, if you are putting batch normalization after a fully connected
layer you should use BATCH_NORM_FC. Otherwise, if you are putting batch
normalization after a convolutional layer you should use BATCH_NORM_CONV.
!*/ !*/
template <typename SUBNET> void setup (const SUBNET& sub); template <typename SUBNET> void setup (const SUBNET& sub);
template <typename SUBNET> void forward(const SUBNET& sub, resizable_tensor& output); void forward_inplace(const tensor& input, tensor& output);
template <typename SUBNET> void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad);
const tensor& get_layer_params() const; const tensor& get_layer_params() const;
tensor& get_layer_params(); tensor& get_layer_params();
/*! /*!
...@@ -656,14 +726,14 @@ namespace dlib ...@@ -656,14 +726,14 @@ namespace dlib
!*/ !*/
}; };
void serialize(const bn_& item, std::ostream& out); void serialize(const affine_& item, std::ostream& out);
void deserialize(bn_& item, std::istream& in); void deserialize(affine_& item, std::istream& in);
/*! /*!
provides serialization support provides serialization support
!*/ !*/
template <typename SUBNET> template <typename SUBNET>
using bn = add_layer<bn_, SUBNET>; using affine = add_layer<affine_, SUBNET>;
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
......
dlib/test/dnn.cpp
View file @ 93ab80c7
...@@ -763,7 +763,101 @@ namespace ...@@ -763,7 +763,101 @@ namespace
DLIB_TEST(max(abs(mat(gamma_grad)-mat(gamma_grad2))) < 1e-4); DLIB_TEST(max(abs(mat(gamma_grad)-mat(gamma_grad2))) < 1e-4);
DLIB_TEST(max(abs(mat(beta_grad)-mat(beta_grad2))) < 1e-4); DLIB_TEST(max(abs(mat(beta_grad)-mat(beta_grad2))) < 1e-4);
} }
#endif
void test_more_ops2()
{
dlib::rand rnd;
tt::tensor_rand trand;
for (int iter = 0; iter < 100; ++iter)
{
print_spinner();
resizable_tensor dest1, dest2, src1, src2;
src1.set_size(rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1);
dest1.copy_size(src1);
dest2.copy_size(src1);
src2.set_size(1,src1.k(),1,1);
trand.fill_uniform(dest1);
trand.fill_uniform(dest2);
trand.fill_uniform(src1);
trand.fill_uniform(src2);
cpu::multiply_conv(dest1, src1, src2);
cuda::multiply_conv(dest2, src1, src2);
DLIB_TEST(max(abs(mat(dest1)-mat(dest2))) < 1e-5);
// now try it using the other mode of multiply_conv
src2.copy_size(src1);
dest1.set_size(1,src1.k(),1,1);
dest2.set_size(1,src1.k(),1,1);
trand.fill_uniform(dest1);
trand.fill_uniform(dest2);
trand.fill_uniform(src1);
trand.fill_uniform(src2);
cpu::multiply_conv(dest1, src1, src2);
cuda::multiply_conv(dest2, src1, src2);
const float scale = max(abs(mat(dest1)));
const float scalem = mean(abs(mat(dest1)));
DLIB_TEST_MSG(max(abs(mat(dest1)-mat(dest2)))/scale < 1e-4 , max(abs(mat(dest1)-mat(dest2)))/scale);
DLIB_TEST_MSG(mean(abs(mat(dest1)-mat(dest2)))/scalem < 1e-5 , mean(abs(mat(dest1)-mat(dest2)))/scalem);
}
for (int iter = 0; iter < 100; ++iter)
{
print_spinner();
resizable_tensor dest1, dest2, src, A, B;
src.set_size(rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1);
dest1.copy_size(src);
dest2.copy_size(src);
A.set_size(1,src.k(),1,1);
B.set_size(1,src.k(),1,1);
trand.fill_uniform(dest1);
trand.fill_uniform(dest2);
trand.fill_uniform(src);
trand.fill_uniform(A);
trand.fill_uniform(B);
cpu::affine_transform_conv(dest1, src, A, B);
cuda::affine_transform_conv(dest2, src, A, B);
DLIB_TEST(max(abs(mat(dest1)-mat(dest2))) < 1e-5);
}
for (int iter = 0; iter < 100; ++iter)
{
print_spinner();
resizable_tensor dest1, dest2, g;
g.set_size(rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1,
rnd.get_random_32bit_number()%30+1);
dest1.set_size(1,g.k(),1,1);
dest2.set_size(1,g.k(),1,1);
trand.fill_uniform(dest1);
trand.fill_uniform(dest2);
trand.fill_uniform(g);
cpu::assign_conv_bias_gradient(dest1, g);
cuda::assign_conv_bias_gradient(dest2, g);
const float scale = max(abs(mat(dest1)));
const float scalem = mean(abs(mat(dest1)));
DLIB_TEST_MSG(max(abs(mat(dest1)-mat(dest2)))/scale < 1e-4 , max(abs(mat(dest1)-mat(dest2)))/scale);
DLIB_TEST_MSG(mean(abs(mat(dest1)-mat(dest2)))/scalem < 1e-5 , mean(abs(mat(dest1)-mat(dest2)))/scalem);
}
}
#endif // DLIB_USE_CUDA
// ---------------------------------------------------------------------------------------- // ----------------------------------------------------------------------------------------
...@@ -883,12 +977,22 @@ namespace ...@@ -883,12 +977,22 @@ namespace
} }
{ {
print_spinner(); print_spinner();
affine_ l; affine_ l(CONV_MODE);
DLIB_TEST_MSG(test_layer(l), test_layer(l));
}
{
print_spinner();
affine_ l(FC_MODE);
DLIB_TEST_MSG(test_layer(l), test_layer(l));
}
{
print_spinner();
bn_ l(CONV_MODE);
DLIB_TEST_MSG(test_layer(l), test_layer(l)); DLIB_TEST_MSG(test_layer(l), test_layer(l));
} }
{ {
print_spinner(); print_spinner();
bn_ l; bn_ l(FC_MODE);
DLIB_TEST_MSG(test_layer(l), test_layer(l)); DLIB_TEST_MSG(test_layer(l), test_layer(l));
} }
{ {
...@@ -953,7 +1057,7 @@ namespace ...@@ -953,7 +1057,7 @@ namespace
template <typename T> using rcon = max_pool<relu<bn<con<T>>>>; template <typename T> using rcon = max_pool<relu<bn<con<T>>>>;
std::tuple<max_pool_,relu_,bn_,con_> rcon_ (unsigned long n) std::tuple<max_pool_,relu_,bn_,con_> rcon_ (unsigned long n)
{ {
return std::make_tuple(max_pool_(2,2,2,2),relu_(),bn_(BATCH_NORM_CONV),con_(n,5,5)); return std::make_tuple(max_pool_(2,2,2,2),relu_(),bn_(CONV_MODE),con_(n,5,5));
} }
template <typename T> using rfc = relu<bn<fc<T>>>; template <typename T> using rfc = relu<bn<fc<T>>>;
...@@ -996,6 +1100,7 @@ namespace ...@@ -996,6 +1100,7 @@ namespace
{ {
test_tagging(); test_tagging();
#ifdef DLIB_USE_CUDA #ifdef DLIB_USE_CUDA
test_more_ops2();
test_more_ops(1,1); test_more_ops(1,1);
test_more_ops(3,4); test_more_ops(3,4);
test_more_ops(4,3); test_more_ops(4,3);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment