[caffe2] Do not run RemoveOpsByType on recurrent networks (pytorch#45986)

Summary:
Pull Request resolved: pytorch#45986

Recurrent networks have subnets that are not well supported by `RemoveOpsByType`. Here we exclude recurrent networks by adding the same check as in memonger.

Test Plan:
```
buck test //caffe2/caffe2/fb/predictor:black_box_predictor_test
```

AdIndexer canary for sanity check:
https://www.internalfb.com/intern/ads/canary/430059485214766620

Differential Revision: D24167284

fbshipit-source-id: fa90d1c1f34af334a599d879af09d4c0bf7c27bd

Author: Hao Lu

caffe2/predictor/transforms.cc

@@ -90,7 +90,7 @@ void RenameOutputs(
 void RenameInputsInChildren(
     const string& from,
     const string& to,
-    std::shared_ptr<caffe2::NetDef> net,
+    caffe2::NetDef* net,
     std::unordered_map<std::string, std::unordered_set<int>>& children) {
   VLOG(2) << "RenameInputsInChildren (from=" << from << ", to=" << to << ")";
   if (children.count(from) == 0) {
@@ -106,7 +106,7 @@ void RenameInputsInChildren(
 void RenameOutputInParents(
     const std::string& from,
     const std::string& to,
-    std::shared_ptr<caffe2::NetDef> net,
+    caffe2::NetDef* net,
     std::unordered_map<std::string, std::unordered_set<int>>& parents) {
   VLOG(2) << "RenameOutputInParents (from=" << from << ", to=" << to << ")";
   if (parents.count(from) == 0) {
@@ -225,7 +225,13 @@ bool FoundOpCandidate(
 // extra complexity is handled in FoundOpCandidate.
 void RemoveOpsByType(InferenceGraph& graph, const std::string& op_type) {
   int num_removed = 0;
-  std::shared_ptr<NetDef> net = graph.predict_net_def;
+  NetDef* net = graph.predict_net_def.get();
+  for (auto& op : net->op()) {
+    if (op.type() == "RecurrentNetwork") {
+      LOG(INFO) << "RemoveOpsByType does not support RecurrentNetwork yet";
+      return;
+    }
+  }
 
   std::unordered_set<std::string> inputs(
       graph.input_names.begin(), graph.input_names.end());
@@ -239,7 +245,7 @@ void RemoveOpsByType(InferenceGraph& graph, const std::string& op_type) {
     for (const auto& o : graph.output_names) {
       net->add_external_output(o);
     }
-    onnx::SsaRewrite(nullptr, net.get());
+    onnx::SsaRewrite(nullptr, net);
     // clear external_outputs
     net->mutable_external_output()->Clear();
     graph.predictor_net_ssa_rewritten = true;