Rename sequence_projection to seq_flow_lite (#9448)

* Rename sequence_projection to seq_flow_lite

* Rename sequence_projection to seq_flow_lite

research/sequence_projection/tf_ops/sequence_string_projection.cc → research/seq_flow_lite/tf_ops/sequence_string_projection.cc

@@ -12,10 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tf_ops/projection_normalizer_util.h"  // sequence_projection
-#include "tf_ops/projection_tokenizer_util.h"  // sequence_projection
-#include "tf_ops/projection_util.h"  // sequence_projection
-#include "tf_ops/text_distorter.h"  // sequence_projection
+#include "tf_ops/projection_normalizer_util.h"  // seq_flow_lite
+#include "tf_ops/projection_tokenizer_util.h"  // seq_flow_lite
+#include "tf_ops/projection_util.h"  // seq_flow_lite
+#include "tf_ops/text_distorter.h"  // seq_flow_lite
 #include "absl/container/flat_hash_map.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/op_kernel.h"

research/sequence_projection/tf_ops/sequence_string_projection_op_v2.cc → research/seq_flow_lite/tf_ops/sequence_string_projection_op_v2.cc

@@ -17,9 +17,9 @@ limitations under the License.
 #include "tensorflow/core/framework/shape_inference.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tf_ops/projection_normalizer_util.h"  // sequence_projection
-#include "tf_ops/projection_util.h"  // sequence_projection
-#include "tf_ops/text_distorter.h"  // sequence_projection
+#include "tf_ops/projection_normalizer_util.h"  // seq_flow_lite
+#include "tf_ops/projection_util.h"  // seq_flow_lite
+#include "tf_ops/text_distorter.h"  // seq_flow_lite
 
 using ::tensorflow::int32;
 using ::tensorflow::int64;
@@ -107,9 +107,9 @@ class SequenceStringProjectionOpV2 : public OpKernel {
     std::vector<uint64_t> hash_codes;
     for (int64 i = 0; i < batch_size; ++i) {
       const int64 num_tokens = seq_len_vector(i);
-      OP_REQUIRES(ctx, num_tokens > 0,
-                  InvalidArgument(
-                      "`sequence_length` should have values greater than 0"));
+      OP_REQUIRES(ctx, num_tokens >= 0,
+                  InvalidArgument("`sequence_length` should have values "
+                                  "greater than or equal to 0"));
       OP_REQUIRES(ctx, num_tokens <= max_seq_len,
                   InvalidArgument("`sequence_length` should have values less "
                                   "than or equal to max_seq_len"));
@@ -181,10 +181,9 @@ REGISTER_OP("SequenceStringProjectionV2")
       const int kMaxFeatureSize = 4096;
       CHECK_GT(feature_size, 0);
       CHECK_LE(feature_size, kMaxFeatureSize);
-      ShapeHandle output_shape;
-      TF_RETURN_IF_ERROR(c->Concatenate(
-          c->input(0), c->MakeShape({feature_size}), &output_shape));
-      c->set_output(0, output_shape);
+      auto batch_size = c->Dim(c->input(0), 0);
+      c->set_output(0, c->MakeShape({batch_size, InferenceContext::kUnknownDim,
+                                     feature_size}));
       return tensorflow::Status::OK();
     })
     .Doc(R"doc(

research/sequence_projection/tf_ops/sequence_string_projection_op_v2_test.cc → research/seq_flow_lite/tf_ops/sequence_string_projection_op_v2_test.cc

@@ -82,7 +82,7 @@ TEST_F(SequenceStringProjectionOpV2Test, TestOutput) {
 
   *mutable_input(1).tensor = Tensor(DT_INT32, TensorShape({2}));
   (*mutable_input(1).tensor).flat<int32>()(0) = 9;
-  (*mutable_input(1).tensor).flat<int32>()(1) = 0;
+  (*mutable_input(1).tensor).flat<int32>()(1) = -1;
 
   EXPECT_EQ(
       RunOpKernel().error_message(),
@@ -91,7 +91,7 @@ TEST_F(SequenceStringProjectionOpV2Test, TestOutput) {
   (*mutable_input(1).tensor).flat<int32>()(0) = 4;
 
   EXPECT_EQ(RunOpKernel().error_message(),
-            "`sequence_length` should have values greater than 0");
+            "`sequence_length` should have values greater than or equal to 0");
 
   (*mutable_input(1).tensor).flat<int32>()(1) = 8;
 

research/sequence_projection/tf_ops/text_distorter.cc → research/seq_flow_lite/tf_ops/text_distorter.cc

@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tf_ops/text_distorter.h"  // sequence_projection
+#include "tf_ops/text_distorter.h"  // seq_flow_lite
 
 using tensorflow::uint32;
 

research/seq_flow_lite/tf_ops/tf_custom_ops.cc

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+
+class PoolingOp : public tensorflow::OpKernel {
+ public:
+  explicit PoolingOp(tensorflow::OpKernelConstruction* context)
+      : tensorflow::OpKernel(context) {}
+
+  void Compute(tensorflow::OpKernelContext* ctx) override {}
+};
+
+REGISTER_KERNEL_BUILDER(Name("PoolingOp").Device(::tensorflow::DEVICE_CPU),
+                        PoolingOp);
+
+REGISTER_OP("PoolingOp")
+    .Input("multiplier: float32")
+    .Input("constant: float32")
+    .Output("state: float32")
+    .Attr("forward: bool")
+    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      return tensorflow::Status::OK();
+    })
+    .Doc(R"doc(
+Dummy pooling op.
+)doc");
+
+class ExpectedValueOp : public tensorflow::OpKernel {
+ public:
+  explicit ExpectedValueOp(tensorflow::OpKernelConstruction* context)
+      : tensorflow::OpKernel(context) {}
+
+  void Compute(tensorflow::OpKernelContext* ctx) override {}
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("ExpectedValueOp").Device(::tensorflow::DEVICE_CPU), ExpectedValueOp);
+
+REGISTER_OP("ExpectedValueOp")
+    .Input("attention_logits: float32")
+    .Input("values: float32")
+    .Output("evalue: float32")
+    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
+      auto batch_size = c->Dim(c->input(0), 0);
+      auto feature_size = c->Dim(c->input(0), 2);
+      c->set_output(0, c->MakeShape({batch_size, feature_size}));
+      return tensorflow::Status::OK();
+    })
+    .Doc(R"doc(
+Dummy pooling op.
+)doc");
+
+class LayerNormOp : public tensorflow::OpKernel {
+ public:
+  explicit LayerNormOp(tensorflow::OpKernelConstruction* context)
+      : tensorflow::OpKernel(context) {}
+
+  void Compute(tensorflow::OpKernelContext* ctx) override {}
+};
+
+REGISTER_KERNEL_BUILDER(Name("LayerNormV2").Device(::tensorflow::DEVICE_CPU),
+                        LayerNormOp);
+
+REGISTER_OP("LayerNormV2")
+    .Input("tensor: float32")
+    .Input("scale: float32")
+    .Input("output: float32")
+    .Attr("axes: list(int)")
+    .Output("result: float32")
+    .SetShapeFn([](::tensorflow::shape_inference::InferenceContext* c) {
+      c->set_output(0, c->input(0));
+      return tensorflow::Status::OK();
+    })
+    .Doc(R"doc(
+Dummy layer norm op.
+)doc");

research/sequence_projection/tflite_ops/BUILD → research/seq_flow_lite/tflite_ops/BUILD

@@ -4,9 +4,7 @@ load("@org_tensorflow//tensorflow/lite:build_def.bzl", "tflite_copts")
 licenses(["notice"])
 
 package(
-    default_visibility = [
-        "//:__subpackages__",  # sequence projection
-    ],
+    default_visibility = ["//:friends"],  # sequence projection
 )
 
 cc_library(
@@ -18,8 +16,8 @@ cc_library(
         ":quantization_util",
         "@org_tensorflow//tensorflow/lite:string_util",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
-        "///tf_ops:projection_normalizer_util",  # sequence projection
-        "///tf_ops:projection_util",  # sequence projection
+        "//tf_ops:projection_normalizer_util",  # sequence projection
+        "//tf_ops:projection_util",  # sequence projection
         "@flatbuffers",
     ],
     alwayslink = 1,
@@ -32,14 +30,14 @@ cc_test(
     deps = [
         ":sequence_string_projection",
         ":tf_tflite_diff_test_util",
+        "@flatbuffers",
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite:string_util",
         "@org_tensorflow//tensorflow/lite/core/api",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
         "@org_tensorflow//tensorflow/lite/kernels:test_util",
-        "///tf_ops:sequence_string_projection_op",  # sequence projection
-        "///tf_ops:sequence_string_projection_op_v2",  # sequence projection
-        "@flatbuffers",
+        # "//tf_ops:sequence_string_projection_op"  # sequence projection
+        "//tf_ops:sequence_string_projection_op_v2",  # sequence projection
     ],
 )
 
@@ -76,3 +74,28 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "layer_norm",
+    srcs = ["layer_norm.cc"],
+    hdrs = ["layer_norm.h"],
+    copts = tflite_copts(),
+    deps = [
+        ":quantization_util",
+        "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
+        "@flatbuffers",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "layer_norm_test",
+    size = "small",
+    srcs = ["layer_norm_test.cc"],
+    deps = [
+        ":layer_norm",
+        "@org_tensorflow//tensorflow/lite/kernels:test_util",
+        "@com_google_googletest//:gtest_main",
+        "@flatbuffers",
+    ],
+)

research/sequence_projection/tflite_ops/expected_value.cc → research/seq_flow_lite/tflite_ops/expected_value.cc

@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tflite_ops/expected_value.h"  // sequence_projection
+#include "tflite_ops/expected_value.h"  // seq_flow_lite
 
 #include <cmath>
 
-#include "tflite_ops/quantization_util.h"  // sequence_projection
+#include "tflite_ops/quantization_util.h"  // seq_flow_lite
 
 namespace tflite {
 namespace ops {

research/seq_flow_lite/tflite_ops/layer_norm.cc

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tflite_ops/layer_norm.h"  // seq_flow_lite
+
+#include <unordered_set>
+#include <vector>
+
+#include "flatbuffers/flexbuffers.h"  // flatbuffer
+#include "tflite_ops/quantization_util.h"  // seq_flow_lite
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+namespace {
+
+const int kInputIndex = 0;
+const int kScaleIndex = 1;
+const int kOffsetIndex = 2;
+const int kAxisIndex = 3;
+const int kOutputIndex = 0;
+
+TfLiteStatus Resize(TfLiteContext* context, TfLiteNode* node) {
+  if (node->outputs->size != 1) {
+    return kTfLiteError;
+  }
+
+  TfLiteTensor* input = &context->tensors[node->inputs->data[kInputIndex]];
+  TfLiteTensor* scale = &context->tensors[node->inputs->data[kScaleIndex]];
+  TfLiteTensor* offset = &context->tensors[node->inputs->data[kOffsetIndex]];
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteUInt8);
+  TF_LITE_ENSURE_EQ(context, offset->dims->data[0], 1);
+  TF_LITE_ENSURE_EQ(context, offset->dims->size, 1);
+  TF_LITE_ENSURE_EQ(context, offset->type, kTfLiteUInt8);
+  TF_LITE_ENSURE_EQ(context, scale->dims->data[0], 1);
+  TF_LITE_ENSURE_EQ(context, scale->dims->size, 1);
+  TF_LITE_ENSURE_EQ(context, scale->type, kTfLiteUInt8);
+  if (node->inputs->size == 4) {
+    TfLiteTensor* axis = &context->tensors[node->inputs->data[kAxisIndex]];
+    TF_LITE_ENSURE_EQ(context, axis->type, kTfLiteInt32);
+  }
+
+  TfLiteTensor* output = &context->tensors[node->outputs->data[kOutputIndex]];
+  TF_LITE_ENSURE_EQ(context, output->type, kTfLiteUInt8);
+  return context->ResizeTensor(context, output,
+                               TfLiteIntArrayCopy(input->dims));
+}
+
+int GetNumberOfSteps(const TfLiteTensor* input) {
+  int number_of_steps = 1;
+  for (int i = 0; i < input->dims->size; ++i) {
+    number_of_steps *= input->dims->data[i];
+  }
+  return number_of_steps;
+}
+
+inline int GetNumberOfFeatures(const TfLiteTensor* input, const int* axis,
+                               const int num_axis) {
+  int num_features = 1;
+  for (int i = 0; i < num_axis; ++i) {
+    num_features *= input->dims->data[axis[i]];
+  }
+  return num_features;
+}
+
+// Performs sanity checks on input axis and resolves into valid dimensions.
+inline bool ResolveAxis(const int num_dims, const int* axis, const int num_axis,
+                        int* out_axis, int* out_num_axis) {
+  *out_num_axis = 0;
+  // Short-circuit axis resolution for scalars; the axis will go unused.
+  if (num_dims == 0) {
+    return true;
+  }
+
+  // Using an unordered set to reduce complexity in looking up duplicates.
+  std::unordered_set<int> unique_indices;
+  for (int64_t idx = 0; idx < num_axis; ++idx) {
+    // Handle negative index.
+    int current = axis[idx] < 0 ? (axis[idx] + num_dims) : axis[idx];
+    assert(current >= 0 && current < num_dims);
+    // Only adding the axis if it wasn't added before.
+    if (unique_indices.find(current) == unique_indices.end()) {
+      unique_indices.insert(current);
+      out_axis[*out_num_axis] = current;
+      *out_num_axis += 1;
+    }
+  }
+  return true;
+}
+
+// Given current position in the input array, the api computes the next valid
+// index.
+bool ValidIndex(const int* input_dims, const int input_dims_size,
+                int* curr_pos) {
+  if (input_dims_size == 0) {
+    return false;
+  }
+  assert(input_dims != nullptr);
+  assert(curr_pos != nullptr);
+  for (int idx = input_dims_size - 1; idx >= 0; --idx) {
+    int current_val = curr_pos[idx] + 1;
+    assert(input_dims[idx] >= current_val);
+    if (input_dims[idx] == current_val) {
+      curr_pos[idx] = 0;
+    } else {
+      curr_pos[idx] = current_val;
+      return true;
+    }
+  }
+  return false;
+}
+
+// Gets next offset depending on reduction axis. Implementation borrowed from
+// tflite reduce mean implementation.
+int GetOffset(const int* input_dims, const int input_dims_size,
+              const int* curr_pos, const int* axis, const int axis_size) {
+  if (input_dims_size == 0) return 0;
+  assert(input_dims != nullptr);
+  assert(curr_pos != nullptr);
+  int offset = 0;
+  for (int idx = 0; idx < input_dims_size; ++idx) {
+    // if idx is part of reduction axes, we skip offset calculation.
+    bool is_axis = false;
+    if (axis != nullptr) {
+      for (int redux = 0; redux < axis_size; ++redux) {
+        if (idx == axis[redux]) {
+          is_axis = true;
+          break;
+        }
+      }
+    }
+    if (!is_axis) offset = offset * input_dims[idx] + curr_pos[idx];
+  }
+
+  return offset;
+}
+
+// TODO(b/132896827): Current implementation needs further evaluation to reduce
+// space time complexities.
+TfLiteStatus FlexibleLayerNorm(const TfLiteTensor* input, const float scale,
+                               const float offset, const int* axis,
+                               const int num_axis, TfLiteTensor* output) {
+  int num_features = GetNumberOfFeatures(input, &axis[0], num_axis);
+  int time_steps = static_cast<int>(GetNumberOfSteps(input) / num_features);
+
+  std::vector<float> sum_x(time_steps, 0.0f);
+  std::vector<float> sum_xx(time_steps, 0.0f);
+  std::vector<int> index_iter(input->dims->size, 0);
+
+  // Computing sum and squared sum for features across the reduction axes.
+  do {
+    // Not passing reduction axes to get the input offset as we are simply
+    // iterating through the multidimensional array.
+    int input_offset = GetOffset(input->dims->data, input->dims->size,
+                                 &index_iter[0], nullptr, 0);
+    // Passing in the valid reduction axes as we would like to get the output
+    // offset after reduction.
+    int stats_offset = GetOffset(input->dims->data, input->dims->size,
+                                 &index_iter[0], &axis[0], num_axis);
+    float input_val = PodDequantize(*input, input_offset);
+    sum_x[stats_offset] += input_val;
+    sum_xx[stats_offset] += input_val * input_val;
+  } while (ValidIndex(input->dims->data, input->dims->size, &index_iter[0]));
+
+  std::vector<float> multiplier(time_steps, 1.0f);
+  std::vector<float> bias(time_steps, 0.0f);
+
+  // Computing stats for the reduction axes.
+  for (int i = 0; i < time_steps; ++i) {
+    sum_x[i] = sum_x[i] / num_features;
+    sum_xx[i] = sum_xx[i] / num_features;
+    const float variance = sum_xx[i] - sum_x[i] * sum_x[i];
+    const float inverse_stddev = 1 / sqrt(variance + 1e-6);
+    multiplier[i] = inverse_stddev * scale;
+    bias[i] = offset - sum_x[i] * inverse_stddev * scale;
+  }
+
+  const float out_inverse_scale = 1.0f / output->params.scale;
+  const int32_t out_zero_point = output->params.zero_point;
+  uint8_t* out_ptr = output->data.uint8;
+  std::fill(index_iter.begin(), index_iter.end(), 0);
+
+  // Using the stats to fill the output pointer.
+  do {
+    // Not passing reduction axes to get the input offset as we are simply
+    // iterating through the multidimensional array.
+    int input_offset = GetOffset(input->dims->data, input->dims->size,
+                                 &index_iter[0], nullptr, 0);
+    // Passing in the valid reduction axes as we would like to get the output
+    // offset after reduction.
+    int stats_offset = GetOffset(input->dims->data, input->dims->size,
+                                 &index_iter[0], &axis[0], num_axis);
+    float input_val = PodDequantize(*input, input_offset);
+
+    const float value =
+        input_val * multiplier[stats_offset] + bias[stats_offset];
+    out_ptr[input_offset] =
+        PodQuantize(value, out_zero_point, out_inverse_scale);
+  } while (ValidIndex(input->dims->data, input->dims->size, &index_iter[0]));
+
+  return kTfLiteOk;
+}
+
+TfLiteStatus DefaultLayerNorm(const TfLiteTensor* input, const float scale,
+                              const float offset, TfLiteTensor* output) {
+  const int input_rank = input->dims->size;
+  const int num_features = input->dims->data[input_rank - 1];
+  const int time_steps =
+      static_cast<int>(GetNumberOfSteps(input) / num_features);
+
+  std::vector<float> temp_buffer(num_features, 0.0f);
+  const float out_inverse_scale = 1.0f / output->params.scale;
+  const int32_t out_zero_point = output->params.zero_point;
+  uint8_t* out_ptr = output->data.uint8;
+  for (int i = 0; i < time_steps; ++i) {
+    float sum_x = 0;
+    float sum_xx = 0;
+    for (int j = 0, index = i * num_features; j < num_features; ++j, ++index) {
+      temp_buffer[j] = PodDequantize(*input, index);
+      sum_x += temp_buffer[j];
+      sum_xx += temp_buffer[j] * temp_buffer[j];
+    }
+    const float exp_xx = sum_xx / num_features;
+    const float exp_x = sum_x / num_features;
+    const float variance = exp_xx - exp_x * exp_x;
+    const float inverse_stddev = 1 / sqrt(variance + 1e-6);
+    const float multiplier = inverse_stddev * scale;
+    const float bias = offset - exp_x * inverse_stddev * scale;
+    for (int j = 0, index = i * num_features; j < num_features; ++j, ++index) {
+      const float value = temp_buffer[j] * multiplier + bias;
+      out_ptr[index] = PodQuantize(value, out_zero_point, out_inverse_scale);
+    }
+  }
+  return kTfLiteOk;
+}
+
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  const TfLiteTensor* input =
+      &context->tensors[node->inputs->data[kInputIndex]];
+  TfLiteTensor* output = &context->tensors[node->outputs->data[kOutputIndex]];
+  const float scale =
+      PodDequantize(context->tensors[node->inputs->data[kScaleIndex]], 0);
+  const float offset =
+      PodDequantize(context->tensors[node->inputs->data[kOffsetIndex]], 0);
+
+  const std::vector<int>& axes =
+      *reinterpret_cast<std::vector<int>*>(node->user_data);
+  const size_t num_axis = axes.size();
+  // For backward compatibility reasons, we handle the default layer norm for
+  // last channel as below.
+  if (num_axis == 1 && (axes[0] == -1 || axes[0] == (input->dims->size - 1))) {
+    return DefaultLayerNorm(input, scale, offset, output);
+  }
+
+  std::vector<int> resolved_axis(num_axis);
+  // Resolve axis.
+  int num_resolved_axis = 0;
+  if (!ResolveAxis(input->dims->size, axes.data(), num_axis, &resolved_axis[0],
+                   &num_resolved_axis)) {
+    return kTfLiteError;
+  }
+
+  return FlexibleLayerNorm(input, scale, offset, &resolved_axis[0],
+                           num_resolved_axis, output);
+}
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+  std::vector<int>* axes = new std::vector<int>();
+  auto axes_fb = m["axes"].AsTypedVector();
+  for (int i = 0; i < axes_fb.size(); ++i) {
+    axes->push_back(axes_fb[i].AsInt32());
+  }
+  return axes;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<std::vector<int>*>(buffer);
+}
+
+}  // namespace
+
+TfLiteRegistration* Register_LAYER_NORM() {
+  static TfLiteRegistration r = {Init, Free, Resize, Eval};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite

research/seq_flow_lite/tflite_ops/layer_norm.h

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef LEARNING_EXPANDER_POD_DEEP_POD_TFLITE_HANDLERS_LAYER_NORM_H_
+#define LEARNING_EXPANDER_POD_DEEP_POD_TFLITE_HANDLERS_LAYER_NORM_H_
+
+#include "tensorflow/lite/kernels/register.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+TfLiteRegistration* Register_LAYER_NORM();
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
+
+#endif  // LEARNING_EXPANDER_POD_DEEP_POD_TFLITE_HANDLERS_LAYER_NORM_H_

research/seq_flow_lite/tflite_ops/layer_norm_test.cc

+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tflite_ops/layer_norm.h"  // seq_flow_lite
+
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"  // flatbuffer
+#include "tensorflow/lite/kernels/test_util.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace {
+
+class LayerNormModel : public SingleOpModel {
+ public:
+  explicit LayerNormModel(const TensorData& input, float output_min,
+                          float output_max, float scale, float offset,
+                          std::initializer_list<int> axis_shape,
+                          std::initializer_list<int> axis)
+      : scale_value_(scale), offset_value_(offset) {
+    input_ = AddInput(input);
+    scale_ = AddInput(
+        {TensorType_UINT8, {1}, std::min(scale, 0.0f), std::max(scale, 0.0f)});
+    offset_ = AddInput({TensorType_UINT8,
+                        {1},
+                        std::min(offset, 0.0f),
+                        std::max(offset, 0.0f)});
+    axis_ = AddConstInput(TensorType_INT32, axis, axis_shape);
+    output_ = AddOutput({TensorType_UINT8, {}, output_min, output_max});
+    flexbuffers::Builder fbb;
+    fbb.Map([&] {
+      {
+        size_t start = fbb.StartVector("axes");
+        for (const int& aval : axis) {
+          fbb.Int(aval);
+        }
+        fbb.EndVector(start, /*typed=*/true, /*fixed=*/false);
+      }
+    });
+    fbb.Finish();
+
+    SetCustomOp("LayerNorm", fbb.GetBuffer(), Register_LAYER_NORM);
+    BuildInterpreter({GetShape(input_)});
+  }
+
+  void SetInput(const std::vector<float>& data) {
+    QuantizeAndPopulate<uint8_t>(input_, data);
+    QuantizeAndPopulate<uint8_t>(scale_, {scale_value_});
+    QuantizeAndPopulate<uint8_t>(offset_, {offset_value_});
+  }
+
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
+                               GetScale(output_), GetZeroPoint(output_));
+  }
+
+ private:
+  int input_;
+  int scale_;
+  int offset_;
+  int axis_;
+  float scale_value_;
+  float offset_value_;
+  int output_;
+};
+
+TEST(LayerNormModelTest, RegularInput) {
+  const float kQuantizedTolerance = 20 * (1. / 256);
+  const std::vector<float> input = {
+      0, -6, 2, 4, 3, -2, 10, 1,
+  };
+  // Mean values are 0.0, 3.0
+  // Standard deviation values are 3.74, 4.41
+  const std::vector<float> expected_output = {0.0, -1.6,  0.53, 1.07,
+                                              0.0, -1.13, 1.59, -0.45};
+
+  LayerNormModel m({TensorType_UINT8, {1, 2, 4}, -10, 10}, -10, 10, 1.0, 0.0,
+                   {1}, {2});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, NegativeScale) {
+  const float kQuantizedTolerance = 20 * (1. / 256);
+  const std::vector<float> input = {
+      0, -6, 2, 4, 3, -2, 10, 1,
+  };
+  // Mean values are 0.0, 3.0
+  // Standard deviation values are 3.74, 4.41
+  const std::vector<float> expected_output = {0.0, 1.6,  -0.53, -1.07,
+                                              0.0, 1.13, -1.59, 0.45};
+  LayerNormModel m({TensorType_UINT8, {1, 2, 4}, -10, 10}, -10, 10, -1.0, 0.0,
+                   {1}, {2});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, NegativeOffset) {
+  const float kQuantizedTolerance = 20 * (1. / 256);
+  const std::vector<float> input = {
+      0, -6, 2, 4, 3, -2, 10, 1,
+  };
+  // Mean values are 0.0, 3.0
+  // Standard deviation values are 3.74, 4.41
+  const std::vector<float> expected_output = {-1.0, -2.6,  -0.53, 0.07,
+                                              -1.0, -2.13, 0.59,  -1.45};
+  LayerNormModel m({TensorType_UINT8, {1, 2, 4}, -10, 10}, -10, 10, 1.0, -1.0,
+                   {1}, {2});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, NegativeScaleAndOffset) {
+  const float kQuantizedTolerance = 20 * (1. / 256);
+  const std::vector<float> input = {
+      0, -6, 2, 4, 3, -2, 10, 1,
+  };
+  // Mean values are 0.0, 3.0
+  // Standard deviation values are 3.74, 4.41
+  const std::vector<float> expected_output = {-1.0, 0.6,  -1.53, -2.07,
+                                              -1.0, 0.13, -2.59, -0.55};
+  LayerNormModel m({TensorType_UINT8, {1, 2, 4}, -10, 10}, -10, 10, -1.0, -1.0,
+                   {1}, {2});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, MultipleAxis) {
+  const float kQuantizedTolerance = 6 * (1. / 256);
+  const std::vector<float> input = {0,  1,  2,  3, 2,  2, 3,  3,  2,  -3, 1, 0,
+                                    -2, -3, -2, 0, -1, 0, -3, -2, -1, 0,  1, 2};
+  const std::vector<float> expected_output = {
+      0.06,  0.57,  1.08,  1.59,  0.69,  0.69,  1.15,  1.15,
+      1.12,  -2.08, 0.48,  -0.16, -0.95, -1.46, -0.95, 0.06,
+      -0.69, -0.23, -1.60, -1.15, -0.80, -0.16, 0.48,  1.12};
+
+  LayerNormModel m({TensorType_UINT8, {1, 2, 3, 4}, -3, 3}, -3, 3, 1.0, 0.0,
+                   {2}, {1, 3});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, MultipleNegativeAxis) {
+  const float kQuantizedTolerance = 6 * (1. / 256);
+  const std::vector<float> input = {0,  1,  2,  3, 2,  2, 3,  3,  2,  -3, 1, 0,
+                                    -2, -3, -2, 0, -1, 0, -3, -2, -1, 0,  1, 2};
+  const std::vector<float> expected_output = {
+      0.06,  0.57,  1.08,  1.59,  0.69,  0.69,  1.15,  1.15,
+      1.12,  -2.08, 0.48,  -0.16, -0.95, -1.46, -0.95, 0.06,
+      -0.69, -0.23, -1.60, -1.15, -0.80, -0.16, 0.48,  1.12};
+
+  LayerNormModel m({TensorType_UINT8, {1, 2, 3, 4}, -3, 3}, -3, 3, 1.0, 0.0,
+                   {2}, {-3, -1});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+TEST(LayerNormModelTest, MultipleAxisWithLargeDepth) {
+  const float kQuantizedTolerance = 7 * (1. / 256);
+  const std::vector<float> input = {
+      0.1, 0.2, 0.3, 0.4, 0.2, 0.3, 0.4, 0.5, 0.1, 0.1, 0.1, 0.1,
+      0.4, 0.2, 0.2, 0.2, 0.9, 0.9, 0.9, 0.9, 0.2, 0.3, 0.7, 0.7,
+      0.1, 0.1, 0.3, 0.3, 0.1, 0.2, 0.3, 0.4, 0.1, 0.2, 0.3, 0.4};
+  const std::vector<float> expected_output = {
+      -1.06, -0.67, -0.28, 0.11,  -0.67, -0.28, 0.11,  0.50,  -1.06,
+      -0.85, -0.85, -0.85, 0.42,  -0.42, -0.42, -0.42, 2.55,  2.55,
+      2.05,  2.05,  -0.67, -0.28, 1.27,  1.27,  -1.06, -1.06, -0.28,
+      0.,    -0.85, -0.42, 0.,    0.42,  -0.85, -0.42, 0.,    0.42};
+
+  LayerNormModel m({TensorType_UINT8, {1, 2, 2, 9}, -1.0, 1.0}, -3.0, 3.0, 1.0,
+                   0.0, {2}, {1, 3});
+  m.SetInput(input);
+  m.Invoke();
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear(expected_output, kQuantizedTolerance)));
+}
+
+}  // namespace
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite

research/sequence_projection/tflite_ops/sequence_string_projection.cc → research/seq_flow_lite/tflite_ops/sequence_string_projection.cc

@@ -15,7 +15,7 @@ limitations under the License.
 /**
  * Sequence String projection op used in PRADO.
  */
-#include "tflite_ops/sequence_string_projection.h"  // sequence_projection
+#include "tflite_ops/sequence_string_projection.h"  // seq_flow_lite
 
 #include <algorithm>
 #include <cmath>
@@ -27,9 +27,9 @@ limitations under the License.
 
 #include "flatbuffers/flexbuffers.h"  // flatbuffer
 #include "tensorflow/lite/string_util.h"
-#include "tf_ops/projection_normalizer_util.h"  // sequence_projection
-#include "tf_ops/projection_util.h"  // sequence_projection
-#include "tflite_ops/quantization_util.h"  // sequence_projection
+#include "tf_ops/projection_normalizer_util.h"  // seq_flow_lite
+#include "tf_ops/projection_util.h"  // seq_flow_lite
+#include "tflite_ops/quantization_util.h"  // seq_flow_lite
 
 namespace tflite {
 namespace ops {

research/sequence_projection/tflite_ops/sequence_string_projection_test.cc → research/seq_flow_lite/tflite_ops/sequence_string_projection_test.cc

@@ -12,11 +12,11 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tflite_ops/sequence_string_projection.h"  // sequence_projection
+#include "tflite_ops/sequence_string_projection.h"  // seq_flow_lite
 
 #include <vector>
 
-#include "tflite_ops/tf_tflite_diff_test_util.h"  // sequence_projection
+#include "tflite_ops/tf_tflite_diff_test_util.h"  // seq_flow_lite
 #include "flatbuffers/flexbuffers.h"  // flatbuffer
 #include "tensorflow/lite/interpreter.h"
 #include "tensorflow/lite/kernels/register.h"

research/sequence_projection/tflite_ops/tf_tflite_diff_test_util.cc → research/seq_flow_lite/tflite_ops/tf_tflite_diff_test_util.cc

@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-#include "tflite_ops/tf_tflite_diff_test_util.h"  // sequence_projection
+#include "tflite_ops/tf_tflite_diff_test_util.h"  // seq_flow_lite
 
 #include "flatbuffers/flexbuffers.h"  // flatbuffer
 #include "tensorflow/core/framework/node_def_builder.h"