code backup

src/include/migraphx/op/capture.hpp

@@ -26,7 +26,7 @@ struct capture
         return pack(f(self.ins_index, "instruction_index"));
     }
 
-    std::string name() const { return "capputure"; }
+    std::string name() const { return "capture"; }
 
     shape compute_shape(std::vector<shape> inputs) const { return inputs.front(); }
 

src/include/migraphx/quantization.hpp

@@ -17,7 +17,10 @@ void quantize(program& prog);
 
 // insert the capture operator for the inputs of each operator to be quantized
 // to int8
-void capture_arguments(program& prog, const std::vector<std::string>& ins_names);
+void capture_arguments(program& prog,
+                       const std::vector<std::string>& ins_names,
+                       std::size_t& num_quant_params,
+                       std::function<void(std::size_t, std::vector<argument> args)> func);
 
 } // namespace MIGRAPHX_INLINE_NS
 } // namespace migraphx

src/quantization.cpp

@@ -2,7 +2,11 @@
 #include <migraphx/program.hpp>
 #include <migraphx/instruction.hpp>
 #include <migraphx/iterator_for.hpp>
-#include <migraphx/op/convert.hpp>
+#include <migraphx/op/dot.hpp>
+#include <migraphx/op/mul.hpp>
+#include <migraphx/op/add.hpp>
+#include <migraphx/op/convolution.hpp>
+#include <migraphx/op/multibroadcast.hpp>
 #include <migraphx/op/capture.hpp>
 #include <migraphx/stringutils.hpp>
 #include <migraphx/ranges.hpp>
@@ -11,25 +15,38 @@
 namespace migraphx {
 inline namespace MIGRAPHX_INLINE_NS {
 
-instruction_ref insert_fp16(program& prog,
+instruction_ref insert_quant_ins(program& prog,
                                  instruction_ref& ins,
                                  shape::type_t type,
-                            std::unordered_map<instruction_ref, instruction_ref>& map_fp16)
+                                 std::unordered_map<instruction_ref, instruction_ref>& map_ins,
+                                 float scale = 1.0f,
+                                 float shift = 0.0f)
 {
-    if(map_fp16.count(ins) > 0)
+    if(map_ins.count(ins) > 0)
     {
-        return map_fp16[ins];
+        return map_ins[ins];
+    }
+
+    if(ins->name() == "undefined")
+    {
+        return ins;
     }
 
     assert(ins->get_shape().type() == shape::float_type ||
-           ins->get_shape().type() == shape::double_type);
-    instruction_ref ins_fp16{};
-    ins_fp16      = prog.insert_instruction(std::next(ins), op::convert{type}, ins);
-    map_fp16[ins] = ins_fp16;
+           ins->get_shape().type() == shape::double_type ||
+           ins->get_shape().type() == shape::int32_type);
+    instruction_ref quant_ins{};
+    quant_ins    = prog.insert_instruction(std::next(ins), op::convert{type, scale, shift}, ins);
+    map_ins[ins] = quant_ins;
 
-    return ins_fp16;
+    return quant_ins;
 }
 
+// This function is to convert any instructions specified in the input
+// from double or float to float16 by inserting a convert operator.
+// For the conversion, there could be cases of overflowing, but it
+// is very rare in the area of deeping learning, so we just do a
+// truncate of the input to get the fp16.
 void quantize(program& prog, const std::vector<std::string>& ins_names)
 {
     std::unordered_map<instruction_ref, instruction_ref> map_fp16;
@@ -60,7 +77,7 @@ void quantize(program& prog, const std::vector<std::string>& ins_names)
                 }
                 else
                 {
-                    input_fp16 = insert_fp16(prog, input, shape::half_type, map_fp16);
+                    input_fp16 = insert_quant_ins(prog, input, shape::half_type, map_fp16);
                 }
                 converted_inputs.push_back(input_fp16);
             }
@@ -79,13 +96,6 @@ void quantize(program& prog, const std::vector<std::string>& ins_names)
         auto op        = ins->get_operator();
         auto ins_shape = compute_shape(op, converted_inputs);
         if(ins_shape.type() != orig_type)
-        {
-            // insert another convert instruction to convert it back
-            if(ins == std::prev(prog.end()))
-            {
-                prog.add_instruction(op::convert{orig_type}, ins);
-            }
-            else
         {
             // check the dead code case to avoid assert
             bool output_empty = ins->outputs().empty();
@@ -96,7 +106,6 @@ void quantize(program& prog, const std::vector<std::string>& ins_names)
                 prog.replace_instruction(ins, ins_orig_type);
             }
         }
-        }
 
         prog.replace_instruction(ins, op, converted_inputs);
     }
@@ -104,30 +113,16 @@ void quantize(program& prog, const std::vector<std::string>& ins_names)
 
 void quantize(program& prog) { quantize(prog, {"all"}); }
 
-std::vector<std::vector<argument>> ins_args;
-void capture_args(std::size_t ins_index, std::vector<argument> args)
-{
-    if(ins_index == ins_args.size())
-    {
-        ins_args.push_back(std::vector<argument>{});
-    }
-    ins_args[ins_index].push_back(args.front());
-
-    return;
-}
-
-void calc_quant_params(std::vector<std::vector<argument>>& ins_arg,
-                       std::vector<std::pair<float, float>>& ins_params)
-{
-    return;
-}
-
 // For the input of each input argument, we need to insert a
 // capture operator to compute the scale and shift
-void capture_arguments(program& prog, const std::vector<std::string>& ins_names)
+void capture_arguments(program& prog,
+                       const std::vector<std::string>& ins_names,
+                       std::size_t& num_quant_params,
+                       std::function<void(std::size_t, std::vector<argument> args)> func)
 {
+    num_quant_params = 0;
     // the int8 quantization only support dot and convolution
-    std::vector<std::string> op_names = {"dot", "convolution"};
+    std::vector<std::string> op_names = {"dot", "convolution", "quant_dot", "quant_convolution"};
     if(!std::all_of(ins_names.begin(), ins_names.end(), [&](auto name) {
            return std::find(op_names.begin(), op_names.end(), name) != op_names.end();
        }))
@@ -136,7 +131,6 @@ void capture_arguments(program& prog, const std::vector<std::string>& ins_names)
     }
 
     std::unordered_map<instruction_ref, instruction_ref> ins_map;
-    std::size_t index = 0;
     for(auto ins : iterator_for(prog))
     {
         if(not contains(ins_names, ins->name()))
@@ -156,7 +150,7 @@ void capture_arguments(program& prog, const std::vector<std::string>& ins_names)
             else
             {
                 new_ins = prog.insert_instruction(
-                    std::next(input), op::capture{index++, capture_args}, input);
+                    std::next(input), op::capture{num_quant_params++, func}, input);
                 ins_map[input] = new_ins;
             }
             new_args.push_back(new_ins);