Skip to content

GitLab

"git@developer.sourcefind.cn:change/sglang.git" did not exist on "8dbdc018a31e1437fb40fec48fd0a9a6ba4fdaee"
Commit af40aa1b authored by Davis King's avatar Davis King
Browse files

Cleaned up how the output is output.

parent 7494f51d
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 100 deletions
tools/convert_dlib_nets_to_caffe/main.cpp
View file @ af40aa1b
......@@ -108,42 +108,48 @@ void convert_dlib_xml_to_cafffe_python_code(
const string& xml_filename
)
{
const string out_filename = left_substr(xml_filename,".") + "_dlib_to_caffe_model.py";
cout << "Writing model to " << out_filename << endl;
ofstream fout(out_filename);
fout.precision(9);
const auto layers = parse_dlib_xml(xml_filename);
cout << "import caffe " << endl;
cout << "from caffe import layers as L, params as P" << endl;
cout << "import numpy as np" << endl;
fout << "import caffe " << endl;
fout << "from caffe import layers as L, params as P" << endl;
fout << "import numpy as np" << endl;
// dlib nets don't commit to a batch size, so just use 1 as the default
cout << "batch_size = 1;" << endl;
fout << "\n# Input tensor dimensions" << endl;
fout << "batch_size = 1;" << endl;
if (layers.back().detail_name == "input_rgb_image")
{
cout << "input_nr = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
cout << "input_nc = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
cout << "input_k = 3;" << endl;
fout << "input_nr = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
fout << "input_nc = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
fout << "input_k = 3;" << endl;
}
else if (layers.back().detail_name == "input_rgb_image_sized")
{
cout << "input_nr = " << layers.back().attribute("nr") << ";" << endl;
cout << "input_nc = " << layers.back().attribute("nc") << ";" << endl;
cout << "input_k = 3;" << endl;
fout << "input_nr = " << layers.back().attribute("nr") << ";" << endl;
fout << "input_nc = " << layers.back().attribute("nc") << ";" << endl;
fout << "input_k = 3;" << endl;
}
else if (layers.back().detail_name == "input")
{
cout << "input_nr = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
cout << "input_nc = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
cout << "input_k = 1;" << endl;
fout << "input_nr = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
fout << "input_nc = 28; #WARNING, the source dlib network didn't commit to a specific input size, so we put 28 here as a default." << endl;
fout << "input_k = 1;" << endl;
}
else
{
throw dlib::error("No known transformation from dlib's " + layers.back().detail_name + " layer to caffe.");
}
fout << endl;
cout << "def make_netspec():" << endl;
cout << " # For reference, the only \"documentation\" about caffe layer parameters seems to be this page:\n";
cout << " # https://github.com/BVLC/caffe/blob/master/src/caffe/proto/caffe.proto\n" << endl;
cout << " n = caffe.NetSpec(); " << endl;
cout << " n.data,n.label = L.MemoryData(batch_size=batch_size, channels=input_k, height=input_nr, width=input_nc, ntop=2)" << endl;
fout << "def make_netspec():" << endl;
fout << " # For reference, the only \"documentation\" about caffe layer parameters seems to be this page:\n";
fout << " # https://github.com/BVLC/caffe/blob/master/src/caffe/proto/caffe.proto\n" << endl;
fout << " n = caffe.NetSpec(); " << endl;
fout << " n.data,n.label = L.MemoryData(batch_size=batch_size, channels=input_k, height=input_nr, width=input_nc, ntop=2)" << endl;
// iterate the layers starting with the input layer
for (auto i = layers.rbegin(); i != layers.rend(); ++i)
{
......@@ -154,33 +160,33 @@ void convert_dlib_xml_to_cafffe_python_code(
if (i->detail_name == "con")
{
cout << " n." << i->caffe_layer_name() << " = L.Convolution(n." << find_input_layer_caffe_name(i);
cout << ", num_output=" << i->attribute("num_filters");
cout << ", kernel_w=" << i->attribute("nc");
cout << ", kernel_h=" << i->attribute("nr");
cout << ", stride_w=" << i->attribute("stride_x");
cout << ", stride_h=" << i->attribute("stride_y");
cout << ", pad_w=" << i->attribute("padding_x");
cout << ", pad_h=" << i->attribute("padding_y");
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.Convolution(n." << find_input_layer_caffe_name(i);
fout << ", num_output=" << i->attribute("num_filters");
fout << ", kernel_w=" << i->attribute("nc");
fout << ", kernel_h=" << i->attribute("nr");
fout << ", stride_w=" << i->attribute("stride_x");
fout << ", stride_h=" << i->attribute("stride_y");
fout << ", pad_w=" << i->attribute("padding_x");
fout << ", pad_h=" << i->attribute("padding_y");
fout << ");\n";
}
else if (i->detail_name == "relu")
{
cout << " n." << i->caffe_layer_name() << " = L.ReLU(n." << find_input_layer_caffe_name(i);
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.ReLU(n." << find_input_layer_caffe_name(i);
fout << ");\n";
}
else if (i->detail_name == "max_pool")
{
cout << " n." << i->caffe_layer_name() << " = L.Pooling(n." << find_input_layer_caffe_name(i);
cout << ", pool=P.Pooling.MAX";
fout << " n." << i->caffe_layer_name() << " = L.Pooling(n." << find_input_layer_caffe_name(i);
fout << ", pool=P.Pooling.MAX";
if (i->attribute("nc")==0)
{
cout << ", global_pooling=True";
fout << ", global_pooling=True";
}
else
{
cout << ", kernel_w=" << i->attribute("nc");
cout << ", kernel_h=" << i->attribute("nr");
fout << ", kernel_w=" << i->attribute("nc");
fout << ", kernel_h=" << i->attribute("nr");
}
if (i->attribute("padding_x") != 0 || i->attribute("padding_y") != 0)
{
......@@ -188,24 +194,24 @@ void convert_dlib_xml_to_cafffe_python_code(
"network with such pooling layers.");
}
cout << ", stride_w=" << i->attribute("stride_x");
cout << ", stride_h=" << i->attribute("stride_y");
cout << ", pad_w=" << i->attribute("padding_x");
cout << ", pad_h=" << i->attribute("padding_y");
cout << ");\n";
fout << ", stride_w=" << i->attribute("stride_x");
fout << ", stride_h=" << i->attribute("stride_y");
fout << ", pad_w=" << i->attribute("padding_x");
fout << ", pad_h=" << i->attribute("padding_y");
fout << ");\n";
}
else if (i->detail_name == "avg_pool")
{
cout << " n." << i->caffe_layer_name() << " = L.Pooling(n." << find_input_layer_caffe_name(i);
cout << ", pool=P.Pooling.AVE";
fout << " n." << i->caffe_layer_name() << " = L.Pooling(n." << find_input_layer_caffe_name(i);
fout << ", pool=P.Pooling.AVE";
if (i->attribute("nc")==0)
{
cout << ", global_pooling=True";
fout << ", global_pooling=True";
}
else
{
cout << ", kernel_w=" << i->attribute("nc");
cout << ", kernel_h=" << i->attribute("nr");
fout << ", kernel_w=" << i->attribute("nc");
fout << ", kernel_h=" << i->attribute("nr");
}
if (i->attribute("padding_x") != 0 || i->attribute("padding_y") != 0)
{
......@@ -213,25 +219,25 @@ void convert_dlib_xml_to_cafffe_python_code(
"network with such pooling layers.");
}
cout << ", stride_w=" << i->attribute("stride_x");
cout << ", stride_h=" << i->attribute("stride_y");
cout << ", pad_w=" << i->attribute("padding_x");
cout << ", pad_h=" << i->attribute("padding_y");
cout << ");\n";
fout << ", stride_w=" << i->attribute("stride_x");
fout << ", stride_h=" << i->attribute("stride_y");
fout << ", pad_w=" << i->attribute("padding_x");
fout << ", pad_h=" << i->attribute("padding_y");
fout << ");\n";
}
else if (i->detail_name == "fc")
{
cout << " n." << i->caffe_layer_name() << " = L.InnerProduct(n." << find_input_layer_caffe_name(i);
cout << ", num_output=" << i->attribute("num_outputs");
cout << ", bias_term=True";
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.InnerProduct(n." << find_input_layer_caffe_name(i);
fout << ", num_output=" << i->attribute("num_outputs");
fout << ", bias_term=True";
fout << ");\n";
}
else if (i->detail_name == "fc_no_bias")
{
cout << " n." << i->caffe_layer_name() << " = L.InnerProduct(n." << find_input_layer_caffe_name(i);
cout << ", num_output=" << i->attribute("num_outputs");
cout << ", bias_term=False";
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.InnerProduct(n." << find_input_layer_caffe_name(i);
fout << ", num_output=" << i->attribute("num_outputs");
fout << ", bias_term=False";
fout << ");\n";
}
else if (i->detail_name == "bn_con" || i->detail_name == "bn_fc")
{
......@@ -240,50 +246,50 @@ void convert_dlib_xml_to_cafffe_python_code(
}
else if (i->detail_name == "affine_con")
{
cout << " n." << i->caffe_layer_name() << " = L.Scale(n." << find_input_layer_caffe_name(i);
cout << ", axis=1";
cout << ", bias_term=True";
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.Scale(n." << find_input_layer_caffe_name(i);
fout << ", axis=1";
fout << ", bias_term=True";
fout << ");\n";
}
else if (i->detail_name == "affine_fc")
{
cout << " n." << i->caffe_layer_name() << " = L.Scale(n." << find_input_layer_caffe_name(i);
cout << ", axis=3";
cout << ", bias_term=True";
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.Scale(n." << find_input_layer_caffe_name(i);
fout << ", axis=3";
fout << ", bias_term=True";
fout << ");\n";
}
else if (i->detail_name == "add_prev")
{
cout << " n." << i->caffe_layer_name() << " = L.Eltwise(n." << find_input_layer_caffe_name(i);
cout << ", n." << find_layer_caffe_name(i, i->attribute("tag"));
cout << ", operation=P.Eltwise.SUM";
cout << ");\n";
fout << " n." << i->caffe_layer_name() << " = L.Eltwise(n." << find_input_layer_caffe_name(i);
fout << ", n." << find_layer_caffe_name(i, i->attribute("tag"));
fout << ", operation=P.Eltwise.SUM";
fout << ");\n";
}
else
{
throw dlib::error("No known transformation from dlib's " + i->detail_name + " layer to caffe.");
}
}
cout << " return n.to_proto();\n\n" << endl;
fout << " return n.to_proto();\n\n" << endl;
// -------------------------
// -------------------------
cout << "def save_as_caffe_model(def_file, weights_file):\n";
cout << " with open(def_file, 'w') as f: f.write(str(make_netspec()));\n";
cout << " net = caffe.Net(def_file, caffe.TEST);\n";
cout << " set_network_weights(net);\n";
cout << " net.save(weights_file);\n\n";
fout << "def save_as_caffe_model(def_file, weights_file):\n";
fout << " with open(def_file, 'w') as f: f.write(str(make_netspec()));\n";
fout << " net = caffe.Net(def_file, caffe.TEST);\n";
fout << " set_network_weights(net);\n";
fout << " net.save(weights_file);\n\n";
// -------------------------
// -------------------------
cout << "def set_network_weights(net):\n";
cout << " # populate network parameters\n";
fout << "def set_network_weights(net):\n";
fout << " # populate network parameters\n";
// iterate the layers starting with the input layer
for (auto i = layers.rbegin(); i != layers.rend(); ++i)
{
......@@ -299,14 +305,14 @@ void convert_dlib_xml_to_cafffe_python_code(
matrix<double> biases = trans(rowm(i->params,range(i->params.size()-num_filters, i->params.size()-1)));
// main filter weights
cout << " p = "; print_as_np_array(cout,weights); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,weights); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
// biases
cout << " p = "; print_as_np_array(cout,biases); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,biases); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
}
else if (i->detail_name == "fc")
{
......@@ -314,23 +320,23 @@ void convert_dlib_xml_to_cafffe_python_code(
matrix<double> biases = rowm(i->params, i->params.nr()-1);
// main filter weights
cout << " p = "; print_as_np_array(cout,weights); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,weights); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
// biases
cout << " p = "; print_as_np_array(cout,biases); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,biases); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
}
else if (i->detail_name == "fc_no_bias")
{
matrix<double> weights = trans(i->params);
// main filter weights
cout << " p = "; print_as_np_array(cout,weights); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,weights); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
}
else if (i->detail_name == "affine_con" || i->detail_name == "affine_fc")
{
......@@ -339,14 +345,14 @@ void convert_dlib_xml_to_cafffe_python_code(
matrix<double> beta = trans(rowm(i->params,range(dims, 2*dims-1)));
// set gamma weights
cout << " p = "; print_as_np_array(cout,gamma); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,gamma); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][0].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][0].data[:] = p;\n";
// set beta weights
cout << " p = "; print_as_np_array(cout,beta); cout << ";\n";
cout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
cout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
fout << " p = "; print_as_np_array(fout,beta); fout << ";\n";
fout << " p.shape = net.params['"<<i->caffe_layer_name()<<"'][1].data.shape;\n";
fout << " net.params['"<<i->caffe_layer_name()<<"'][1].data[:] = p;\n";
}
}
......@@ -356,8 +362,13 @@ void convert_dlib_xml_to_cafffe_python_code(
int main(int argc, char** argv) try
{
cout.precision(9);
// TODO, write out to multiple files or just process one file at a time.
if (argc == 1)
{
cout << "Give this program an xml file generated by dlib::net_to_xml() and it will" << endl;
cout << "convert it into a python file that outputs a caffe model containing the dlib model." << endl;
return 0;
}
for (int i = 1; i < argc; ++i)
convert_dlib_xml_to_cafffe_python_code(argv[i]);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment