Mapped bidirectional lstm & unit test

include/tim/vx/ops/bidirectional_sequence_lstm.h

@@ -0,0 +1,67 @@
+/****************************************************************************
+*
+*    Copyright (c) 2022 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************/
+#ifndef TIM_VX_OPS_BIDIRECTIONAL_SEQUENCE_LSTM_H_
+#define TIM_VX_OPS_BIDIRECTIONAL_SEQUENCE_LSTM_H_
+
+#include "tim/vx/operation.h"
+namespace tim {
+namespace vx {
+namespace ops {
+
+class BidirectionalSequenceLstm : public Operation {
+ public:
+  enum ActivationType {
+    kNONE = 0,
+    kRELU = 1,
+    kRELU1 = 2,
+    kRELU6 = 3,
+    kTANH = 4,
+    kSIGMOID = 6,
+    kHARDSIGMOID = 31, /* temporary use 31 */
+  };
+  BidirectionalSequenceLstm(
+          Graph* graph, float cell_clip, float proj_clip,
+          ActivationType act_type, float forget_bias, bool time_major = false,
+          ActivationType recurrent_act_type = ActivationType::kSIGMOID,
+          bool return_sequences = false /*False: only return last state*/
+      );
+
+  std::shared_ptr<Operation> Clone(
+      std::shared_ptr<Graph>& graph) const override;
+
+ protected:
+  const float cell_clip_;
+  const float proj_clip_;
+  const ActivationType act_type_;
+  const float forget_bias_;
+  const bool time_major_;
+  const ActivationType recurrent_act_type_;
+  const bool return_sequences_;
+};
+
+}  // namespace ops
+}  // namespace vx
+}  // namespace tim
+
+#endif /* TIM_VX_OPS_BIDIRECTIONAL_SEQUENCE_LSTM_H_ */

src/tim/vx/ops/README.md

@@ -109,13 +109,13 @@ GroupedConv1d|GROUPED_CONV1D|Mapped|[tf.keras.layers.Conv1D](https://tensorflow.
 |BroadCast|EXPAND_BROADCAST|Mapped|[numpy.broadcast_to](https://numpy.org/doc/stable/reference/generated/numpy.broadcast_to.html)
 ||PROPOSAL| TBD |[Faster-RCNN Proposal Layer](https://github.com/intel/caffe/blob/master/examples/faster-rcnn/lib/rpn/proposal_layer.py)
 ||ROI_POOL|Planned 22Q4|[ANEURALNETWORKS_ROI_POOLING](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a6736198af337b2efbdb0b6b64dee7fe4)
-ROI_Align||ROI_ALIGN|Mapped|[ANEURALNETWORKS_ROI_ALIGN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a2848b39dd4bfba78f2438fda0d9397a4)
-TopK||TOPK|Mapped (limited support)|[tf.math.top_k](https://tensorflow.google.cn/api_docs/python/tf/math/top_k)
+|ROI_Align|ROI_ALIGN|Mapped|[ANEURALNETWORKS_ROI_ALIGN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a2848b39dd4bfba78f2438fda0d9397a4)
+|TopK|TOPK|Mapped (limited support)|[tf.math.top_k](https://tensorflow.google.cn/api_docs/python/tf/math/top_k)
 |GRUCell|GRUCELL_OVXLIB|Mapped|[tf.keras.layers.GRUCell](https://tensorflow.google.cn/api_docs/python/tf/keras/layers/GRUCell?hl=en)
 |UnidirectionalSequenceGRU|GRU_OVXLIB|Planned 22Q3|[tf.keras.layers.GRU](https://tensorflow.google.cn/api_docs/python/tf/keras/layers/GRUCell?hl=en)
 |UnidirectionalSequenceRNN|UNIDIRECTIONAL_SEQUENCE_RNN|Planned 22Q3|[ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_RNN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0ae11aa1d461d2abaa117f6ee2cb503dd8)
 |BidirectionalSequenceRNN|BIDIRECTIONAL_SEQUENCE_RNN|Planned 22Q3|[ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_RNN](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a487fc5ae247de828f13e62b99f259f3c)
-|BidirectionalSequenceLSTM|BIDIRECTIONAL_SEQUENCE_LSTM|Planned 22Q3|[ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_LSTM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a492a71cb7aa50b9a1a834a3cb269d778)
+|BidirectionalSequenceLSTM|BIDIRECTIONAL_SEQUENCE_LSTM|Mapped|[ANEURALNETWORKS_BIDIRECTIONAL_SEQUENCE_LSTM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0a492a71cb7aa50b9a1a834a3cb269d778)
 |UnidirectionalSequenceLSTM|LSTM_OVXLIB|Mapped|[ANEURALNETWORKS_UNIDIRECTIONAL_SEQUENCE_LSTM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0aaf30e491ad0b1fc7602cbde695b2c859)
 |LSTMCell|LSTMUNIT_OVXLIB|replace with UnidirectionalSequenceLSTM by set n_step = 1 |[ANEURALNETWORKS_LSTM](https://developer.android.com/ndk/reference/group/neural-networks#group___neural_networks_1ggaabbe492c60331b13038e39d4207940e0ad0377e8c305e596fb7f64ff896671fc5)
 ||PRE_PROCESS|TBD |Image Preprocessing (YUV2RGB, Input Normalization, Resizing, etc)

src/tim/vx/ops/bidirectional_sequence_lstm.cc

@@ -0,0 +1,268 @@
+/****************************************************************************
+*
+*    Copyright (c) 2022 Vivante Corporation
+*
+*    Permission is hereby granted, free of charge, to any person obtaining a
+*    copy of this software and associated documentation files (the "Software"),
+*    to deal in the Software without restriction, including without limitation
+*    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+*    and/or sell copies of the Software, and to permit persons to whom the
+*    Software is furnished to do so, subject to the following conditions:
+*
+*    The above copyright notice and this permission notice shall be included in
+*    all copies or substantial portions of the Software.
+*
+*    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+*    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+*    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+*    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+*    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+*    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+*    DEALINGS IN THE SOFTWARE.
+*
+*****************************************************************************/
+#include "tim/vx/ops/bidirectional_sequence_lstm.h"
+#include "tim/vx/ops/unidirectional_sequence_lstm.h"
+#include "vsi_nn_pub.h"
+#include "op_impl.h"
+
+#include <array>
+namespace tim {
+namespace vx {
+namespace ops {
+
+class BidirectionalSequenceLstmImpl : public OpImpl {
+ public:
+  enum {
+    BI_LSTM_INPUT_INPUT = 0,
+
+    BI_LSTM_FW_INPUT_WEIGHT_I2I = 1,
+    BI_LSTM_FW_INPUT_WEIGHT_I2F = 2,
+    BI_LSTM_FW_INPUT_WEIGHT_I2C = 3,
+    BI_LSTM_FW_INPUT_WEIGHT_I2O = 4,
+
+    BI_LSTM_FW_INPUT_WEIGHT_R2I = 5,
+    BI_LSTM_FW_INPUT_WEIGHT_R2F = 6,
+    BI_LSTM_FW_INPUT_WEIGHT_R2C = 7,
+    BI_LSTM_FW_INPUT_WEIGHT_R2O = 8,
+
+    BI_LSTM_FW_INPUT_WEIGHT_C2I = 9,
+    BI_LSTM_FW_INPUT_WEIGHT_C2F = 10,
+    BI_LSTM_FW_INPUT_WEIGHT_C2O = 11,
+
+    BI_LSTM_FW_INPUT_BIAS_I = 12,
+    BI_LSTM_FW_INPUT_BIAS_F = 13,
+    BI_LSTM_FW_INPUT_BIAS_C = 14,
+    BI_LSTM_FW_INPUT_BIAS_O = 15,
+
+    BI_LSTM_FW_INPUT_WEIGHT_PROJ = 16,
+    BI_LSTM_FW_INPUT_BIAS_PROJ = 17,
+
+    BI_LSTM_BW_INPUT_WEIGHT_I2I = 18,
+    BI_LSTM_BW_INPUT_WEIGHT_I2F = 19,
+    BI_LSTM_BW_INPUT_WEIGHT_I2C = 20,
+    BI_LSTM_BW_INPUT_WEIGHT_I2O = 21,
+
+    BI_LSTM_BW_INPUT_WEIGHT_R2I = 22,
+    BI_LSTM_BW_INPUT_WEIGHT_R2F = 23,
+    BI_LSTM_BW_INPUT_WEIGHT_R2C = 24,
+    BI_LSTM_BW_INPUT_WEIGHT_R2O = 25,
+
+    BI_LSTM_BW_INPUT_WEIGHT_C2I = 26,
+    BI_LSTM_BW_INPUT_WEIGHT_C2F = 27,
+    BI_LSTM_BW_INPUT_WEIGHT_C2O = 28,
+
+    BI_LSTM_BW_INPUT_BIAS_I = 29,
+    BI_LSTM_BW_INPUT_BIAS_F = 30,
+    BI_LSTM_BW_INPUT_BIAS_C = 31,
+    BI_LSTM_BW_INPUT_BIAS_O = 32,
+
+    BI_LSTM_BW_INPUT_WEIGHT_PROJ = 33,
+    BI_LSTM_BW_INPUT_BIAS_PROJ = 34,
+
+    BI_LSTM_FW_INPUT_H_STATE = 35,
+    BI_LSTM_FW_INPUT_C_STATE = 36,
+
+    BI_LSTM_BW_INPUT_H_STATE = 37,
+    BI_LSTM_BW_INPUT_C_STATE = 38,
+
+    BI_LSTM_AUX_INPUT = 39,
+
+    BI_LSTM_FW_AUX_INPUT_WEIGHT_I2I = 40,
+    BI_LSTM_FW_AUX_INPUT_WEIGHT_I2F = 41,
+    BI_LSTM_FW_AUX_INPUT_WEIGHT_I2C = 42,
+    BI_LSTM_FW_AUX_INPUT_WEIGHT_I2O = 43,
+
+    BI_LSTM_BW_AUX_INPUT_WEIGHT_I2I = 44,
+    BI_LSTM_BW_AUX_INPUT_WEIGHT_I2F = 45,
+    BI_LSTM_BW_AUX_INPUT_WEIGHT_I2C = 46,
+    BI_LSTM_BW_AUX_INPUT_WEIGHT_I2O = 47,
+
+    BI_LSTM_FW_INPUT_LAYERNORM_I = 48,
+    BI_LSTM_FW_INPUT_LAYERNORM_F = 49,
+    BI_LSTM_FW_INPUT_LAYERNORM_C = 50,
+    BI_LSTM_FW_INPUT_LAYERNORM_O = 51,
+
+    BI_LSTM_BW_INPUT_LAYERNORM_I = 52,
+    BI_LSTM_BW_INPUT_LAYERNORM_F = 53,
+    BI_LSTM_BW_INPUT_LAYERNORM_C = 54,
+    BI_LSTM_BW_INPUT_LAYERNORM_O = 55,
+
+    INPUT_CNT,
+
+    BI_LSTM_FW_OUTPUT_OUTPUT = 0,
+    BI_LSTM_FW_OUTPUT_H_STATE = 1,
+    BI_LSTM_FW_OUTPUT_C_STATE = 2,
+
+    BI_LSTM_BW_OUTPUT_OUTPUT = 3,
+    BI_LSTM_BW_OUTPUT_H_STATE = 4,
+    BI_LSTM_BW_OUTPUT_C_STATE = 5,
+
+    OUTPUT_CNT
+  };
+
+  BidirectionalSequenceLstmImpl(Graph* graph, int input_cnt, int output_cnt,
+                                DataLayout layout = DataLayout::ANY)
+      : OpImpl(graph, -1, input_cnt, output_cnt, layout) {
+    lstm_forward_ = graph->CreateOperation<UnidirectionalSequenceLstm>(
+        0.0, 0.0, UnidirectionalSequenceLstm::kTANH, 0.0, false,
+        UnidirectionalSequenceLstm::kSIGMOID, true);
+    lstm_backward_ =
+        graph->CreateOperation<tim::vx::ops::UnidirectionalSequenceLstm>(
+            0.0, 0.0, UnidirectionalSequenceLstm::kTANH, 0.0, false,
+            UnidirectionalSequenceLstm::kSIGMOID, true);
+  }
+
+  ~BidirectionalSequenceLstmImpl() {}
+
+  BidirectionalSequenceLstmImpl& BindInput(
+      const std::shared_ptr<Tensor>& tensor) override {
+    in_tensors_[input_tensor_index] = tensor;
+
+    if (this->input_tensor_index == INPUT_CNT - 1) {
+      // Get all input tensor
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_INPUT_INPUT]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_H_STATE]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_C_STATE]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_I2I]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_I2F]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_I2C]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_I2O]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_R2I]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_R2F]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_R2C]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_R2O]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_C2I]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_C2F]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_C2O]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_BIAS_I]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_BIAS_F]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_BIAS_C]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_BIAS_O]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_WEIGHT_PROJ]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_BIAS_PROJ]);
+
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_LAYERNORM_I]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_LAYERNORM_F]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_LAYERNORM_C]);
+      lstm_forward_->BindInput(in_tensors_[BI_LSTM_FW_INPUT_LAYERNORM_O]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_INPUT_INPUT]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_H_STATE]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_C_STATE]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_I2I]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_I2F]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_I2C]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_I2O]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_R2I]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_R2F]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_R2C]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_R2O]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_C2I]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_C2F]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_C2O]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_BIAS_I]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_BIAS_F]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_BIAS_C]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_BIAS_O]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_WEIGHT_PROJ]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_BIAS_PROJ]);
+
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_LAYERNORM_I]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_LAYERNORM_F]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_LAYERNORM_C]);
+      lstm_backward_->BindInput(in_tensors_[BI_LSTM_BW_INPUT_LAYERNORM_O]);
+    }
+    this->input_tensor_index++;
+    return *this;
+  }
+
+  BidirectionalSequenceLstmImpl& BindOutput(
+      const std::shared_ptr<Tensor>& tensor) override {
+    out_tensors_[output_tensor_index] = tensor;
+
+    if (this->output_tensor_index == OUTPUT_CNT - 1) {
+      lstm_forward_->BindOutput(out_tensors_[BI_LSTM_FW_OUTPUT_OUTPUT]);
+      lstm_forward_->BindOutput(out_tensors_[BI_LSTM_FW_OUTPUT_H_STATE]);
+      lstm_forward_->BindOutput(out_tensors_[BI_LSTM_FW_OUTPUT_C_STATE]);
+
+      lstm_backward_->BindOutput(out_tensors_[BI_LSTM_BW_OUTPUT_OUTPUT]);
+      lstm_backward_->BindOutput(out_tensors_[BI_LSTM_BW_OUTPUT_H_STATE]);
+      lstm_backward_->BindOutput(out_tensors_[BI_LSTM_BW_OUTPUT_C_STATE]);
+    }
+    this->output_tensor_index++;
+    return *this;
+  }
+
+  vsi_nn_node_t* node() override { return nullptr; }
+
+  std::vector<std::shared_ptr<Tensor>> InputsTensor() override {
+    return inputs_tensor_;
+  }
+  std::vector<std::shared_ptr<Tensor>> OutputsTensor() override {
+    return outputs_tensor_;
+  }
+
+ private:
+  std::shared_ptr<tim::vx::Operation> lstm_forward_;
+  std::shared_ptr<tim::vx::Operation> lstm_backward_;
+
+  std::array<std::shared_ptr<tim::vx::Tensor>, INPUT_CNT> in_tensors_;
+  std::array<std::shared_ptr<tim::vx::Tensor>, OUTPUT_CNT> out_tensors_;
+};
+
+BidirectionalSequenceLstm::BidirectionalSequenceLstm(
+    Graph* graph, float cell_clip, float proj_clip, ActivationType act_type,
+    float forget_bias, bool time_major, ActivationType recurrent_act_type,
+    bool return_sequences)
+    : cell_clip_(cell_clip),
+      proj_clip_(proj_clip),
+      act_type_(act_type),
+      forget_bias_(forget_bias),
+      time_major_(time_major),
+      recurrent_act_type_(recurrent_act_type),
+      return_sequences_(return_sequences) {
+  impl_ = std::make_unique<BidirectionalSequenceLstmImpl>(graph, 0, 0,
+                                                          DataLayout::ANY);
+}
+
+std::shared_ptr<Operation> BidirectionalSequenceLstm::Clone(
+    std::shared_ptr<Graph>& graph) const {
+  return graph->CreateOperation<BidirectionalSequenceLstm>(
+      this->cell_clip_, this->proj_clip_, this->act_type_, this->forget_bias_,
+      this->time_major_, this->recurrent_act_type_, this->return_sequences_);
+}
+
+}  // namespace ops
+}  // namespace vx
+}  // namespace tim